Release v0.5.1

Skip download on archlinux if desired packages are installed

.travis.yml

@@ -0,0 +1,57 @@
+language: rust
+
+cache: cargo
+
+matrix:
+  include:
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=test
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=common
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=boxxy
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=docker
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=docker-registry
+  - os: osx
+    rust: stable
+    env:
+    - BUILD_MODE=test
+  - os: osx
+    rust: stable
+    env:
+    - BUILD_MODE=common
+  #- os: windows
+  #  rust: stable
+  #  env:
+  #  - BUILD_MODE="windows test"
+  - os: windows
+    rust: stable
+    env:
+    - BUILD_MODE="windows common"
+
+before_install:
+- ci/setup.sh "$TRAVIS_OS_NAME"
+script:
+- df -h
+- ci/run.sh $BUILD_MODE
+- df -h
+
+notifications:
+  irc:
+    channels:
+    - "ircs://irc.hackint.org:6697/#sn0int"
+    #on_success: change # default: always
+    #on_failure: always # default: always
+    use_notice: true

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.1.0"
+version = "0.5.1"
 description = "OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -8,19 +8,22 @@ repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
 
+[badges]
+travis-ci = { repository = "kpcyrd/sn0int" }
+
 [workspace]
 members = ["sn0int-registry/sn0int-common",
            "sn0int-registry"]
 
 [dependencies]
-sn0int-common = { version="0.1.0", path="sn0int-registry/sn0int-common" }
+sn0int-common = { version="0.3.0", path="sn0int-registry/sn0int-common" }
 rustyline = "2"
 log = "0.4"
-env_logger = "0.5"
+env_logger = "0.6"
 hlua-badtouch = "0.4"
 structopt = "0.2"
 failure = "0.1"
-rand = "0.5"
+rand = "0.6"
 colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
@@ -29,22 +32,38 @@ diesel = { version = "1.0.0", features = ["sqlite"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
 dirs = "1.0"
 url = "1.7"
-nix = "0.11"
-caps = "0.3"
-syscallz = "0.7"
-chrootable-https = "0.3.3"
-trust-dns-proto = "0.5.0-alpha.1"
-base64 = "0.9"
+chrootable-https = "0.3.4"
+trust-dns-proto = "0.5.0"
+base64 = "0.10"
 kuchiki = "0.7.2"
 serde_urlencoded = "0.5"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-crossbeam-channel = "0.2"
+crossbeam-channel = "0.3"
 ctrlc = "3.1"
 opener = "0.3.0"
 separator = "0.3.1"
 maplit = "1.0.1"
+sloppy-rfc4880 = "0.1.2"
+regex = "1.0"
+toml = "0.4"
+maxminddb = "0.10.0"
+tar = "0.4.17"
+libflate = "0.1.14"
+threadpool = "1.7"
+x509-parser = "0.4.0"
+der-parser = "1.1.0"
+nom = "4.1.1"
+
+[target.'cfg(target_os="linux")'.dependencies]
+caps = "0.3"
+syscallz = "0.7"
+nix = "0.11"
+
+[target.'cfg(target_os="openbsd")'.dependencies]
+pledge = "0.3.1"
+unveil = "0.2.0"
 
 [dev-dependencies]
-boxxy = "0.7"
+boxxy = "0.8"

README.md

@@ -1,6 +1,44 @@
-# sn0int
+# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs]
 
-work in progress
+[travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
+[travis]:       https://travis-ci.org/kpcyrd/sn0int
+[crates-img]:   https://img.shields.io/crates/v/sn0int.svg
+[crates]:       https://crates.io/crates/sn0int
+[docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
+[docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
+
+sn0int is an OSINT framework and package manager. It was built for IT security
+professionals and bug hunters to gather intelligence about a given target or
+about yourself. sn0int is enumerating attack surface by semi-automatically
+processing public information and mapping the results in a unified format for
+followup investigations.
+
+Among other things, sn0int is currently able to:
+
+- [X] Harvest subdomains from certificate transparency logs
+- [X] Harvest subdomains from various passive dns logs
+- [X] Sift through subdomain results for publicly accessible websites
+- [X] Harvest emails from pgp keyservers
+- [X] Enrich ip addresses with ASN and geoip info
+- [X] Harvest subdomains from the wayback machine
+
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource.  None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
+
+Join us on IRC: <ircs://irc.hackint.org/#sn0int>
+
+## Installation
+
+- Archlinux: `yaourt -S sn0int`
+- Alpine: `apk add --no-cache sqlite-dev libseccomp-dev cargo` + build from source
+- Debian: `apt install libsqlite3-dev libseccomp-dev` + build from source
+- OpenBSD: `pkg_add sqlite3` + build from source
+- OSX: `brew install sqlite3` + build from source
 
 ## License
 

ci/bench.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+set -eu
+X=$(mktemp -d)
+cd "$X"
+
+mkdir -p "$X/.cache"
+cp -r "$HOME/.cache/sn0int" "$X/.cache/"
+
+#export CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
+#export RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}"
+export HOME="$X"
+
+cat > 1k.lua <<EOF
+-- Description: Insert 1k random subdomains
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run(arg)
+    for i=1,1000 do
+        x = 'foo' .. i .. '.example.com'
+
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=x,
+        })
+    end
+end
+EOF
+
+echo '[*] Setting up workspace'
+echo 'add domain example.com' | "$@" > /dev/null
+echo '[*] Running 1k inserts'
+time "$@" run -f ./1k.lua

ci/boxxy_stage1.txt

@@ -0,0 +1,3 @@
+stage1
+ls
+echo sandbox fail

ci/reprotest.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -xue
+
+# tested with rustc 1.30.0 and cargo 1.30.0
+
+# by default, the build folder is located in /tmp, which is a tmpfs. The target/ folder
+# can become quite large, causing the build to fail if we don't have enough RAM.
+export TMPDIR="$HOME/tmp/repro-test"
+mkdir -p "$TMPDIR"
+
+# fileordering: https://github.com/diesel-rs/diesel/pull/1902
+# build_path: https://github.com/briansmith/ring/issues/715
+
+reprotest -vv --vary=-time,-domain_host,-fileordering,-build_path --source-pattern 'Cargo.* src/ sn0int-registry/ migrations/' '
+    CARGO_HOME="$PWD/.cargo" RUSTUP_HOME='"$HOME/.rustup"' \
+        RUSTFLAGS="--remap-path-prefix=$HOME=/remap-home --remap-path-prefix=$PWD=/remap-pwd" \
+        cargo build --release --verbose --locked' \
+    target/release/sn0int

ci/run.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+set -exu
+case "$1" in
+    build)
+        cargo build --verbose
+        cargo build --verbose --examples
+        ;;
+    test)
+        ci/run.sh build
+        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
+             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
+        cargo run --example maxmind-dl -- -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
+        cargo run --example maxmind-dl -- -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
+        cargo test --verbose
+        cargo test --verbose -- --ignored
+        ;;
+    common)
+        cd sn0int-registry/sn0int-common
+        cargo test --verbose
+        ;;
+    windows)
+        export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
+        ci/run.sh "$2"
+        ;;
+    boxxy)
+        cargo build --verbose --examples
+        if cat ci/boxxy_stage1.txt | RUST_LOG=boxxy cargo run --example boxxy; then
+            echo "SANDOX ERROR: should've crashed"
+            exit 1
+        fi
+        ;;
+    docker)
+        docker build -t sn0int .
+        docker images
+        docker run --rm sn0int --help
+        ;;
+    docker-registry)
+        docker build -t sn0int-registry sn0int-registry/
+        docker images
+        ;;
+esac

ci/setup.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+set -exu
+case "$1" in
+    linux)
+        sudo apt update
+        sudo apt install libsqlite3-dev libseccomp-dev
+        ;;
+    osx)
+        brew install sqlite3
+        ;;
+    windows)
+        curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
+        7z e sqlite3.zip -y
+        "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
+        ;;
+esac

docs/.gitignore

@@ -0,0 +1 @@
+/_build/

docs/Makefile

@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'sn0int'
+copyright = '2018, kpcyrd'
+author = 'kpcyrd'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = ''
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'sn0intdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'sn0int.tex', 'sn0int Documentation',
+     'kpcyrd', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('man', 'sn0int', 'OSINT framework and package manager',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'sn0int', 'sn0int Documentation',
+     author, 'sn0int', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']

docs/database.rst

@@ -0,0 +1,84 @@
+Database
+========
+
+There are a few things you need to understand how the database works to use it
+efficiently.
+
+The database that is backing sn0int is sqlite, but the api that is exposed to
+the user and scripts is an nosql-ish object store. The query language that is
+exposed to the user is still very similar to sql, except that it lacks a column
+statement::
+
+    select subdomains where value like %.example.com
+    ^      ^          ^           ^    ^
+    |      |          |           |    this value is going to be quoted automatically
+    |      |          |           |
+    |      |          |           this triggers automatic quoting
+    |      |          |
+    |      |          apply a filter, this translates to sql quite literally
+    |      |
+    |      the entity we want to select is a subdomain.
+    |      this affects the table and the deserializer
+    |
+    select entities
+
+This is how almost all user facing functions work that operate on the database.
+The functions that are available for scripting are a bit more object based and
+described below.
+
+db_add
+------
+
+This operation is somewhat straight forward. It adds an entity to the
+database:
+
+.. code-block:: lua
+
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+
+If this entity conflicts with an entity that already exists, an upsert is
+triggered and an db_update is performed instead.
+
+.. note::
+   This function may return ``nil`` if the entity already exists, but has been
+   removed from scope with ``noscope``. Everytime you use ``db_add`` you need
+   to make sure that the ID that has been returned is not ``nil``.
+
+db_update
+---------
+
+Update some mutable fields of an entity:
+
+.. code-block:: lua
+
+    db_update('ipaddr', arg, {
+        asn=lookup['asn'],
+        as_org=lookup['as_org'],
+    })
+
+The first parameter is usually the same arg that your script was called with.
+Usually you can use db_add instead of db_update due to the upsert feature, but
+db_update is still slightly faster.
+
+.. note::
+   Some fields are immutable and can not be updated.
+
+db_select
+---------
+
+This function is used to check if something is in scope. If the entity has been
+added to the database and has not been removed from scope, this function
+returns that entities id. This is somewhat similar to ``db_add``, except that
+``db_select`` never adds anything to the database.
+
+.. code-block:: lua
+
+    domain_id = db_select('domain', 'example.com')
+    if domain_id ~= nil then
+        -- do something
+    end
+
+This function only accepts a string instead of a lua table. This string is used
+to filter on the ``value`` column.

docs/index.rst

@@ -0,0 +1,40 @@
+sn0int
+======
+
+sn0int is an OSINT framework and package manager. It was built for IT security
+professionals and bug hunters to gather intelligence about a given target or
+about yourself. sn0int is enumerating attack surface by semi-automatically
+processing public information and mapping the results in a unified format for
+followup investigations.
+
+Among other things, sn0int is currently able to:
+
+- [X] Harvest subdomains from certificate transparency logs
+- [X] Harvest subdomains from various passive dns logs
+- [X] Sift through subdomain results for publicly accessible websites
+- [X] Harvest emails from pgp keyservers
+- [X] Enrich ip addresses with ASN and geoip info
+- [X] Harvest subdomains from the wayback machine
+
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource.  None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
+
+Join us on IRC: ircs://irc.hackint.org/#sn0int
+
+Getting Started
+---------------
+
+.. toctree::
+   :maxdepth: 3
+   :glob:
+
+   install
+   usage
+   scripting
+   database
+   reference

docs/install.rst

@@ -0,0 +1,66 @@
+Installation
+============
+
+If available, please prefer the package shipped by your linux distribution.
+
+Archlinux
+---------
+
+.. code-block:: bash
+
+    $ yaourt -S sn0int
+
+Debian/Ubuntu/Kali
+------------------
+
+.. code-block:: bash
+
+    $ apt install libsqlite3-dev libseccomp-dev
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f
+
+Alpine
+------
+
+.. code-block:: bash
+
+    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f
+
+OpenBSD
+-------
+
+.. code-block:: bash
+
+    $ pkg_add sqlite3
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f
+
+Mac OSX
+-------
+
+.. code-block:: bash
+
+    $ brew install sqlite3
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f
+
+Windows
+-------
+
+This is not recommended and only passively maintained. Please prefer linux in a virtual machine if needed.
+
+.. code-block:: bash
+
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
+    $ 7z e sqlite3.zip -y
+    $ "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
+    $ export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
+    $ cargo install -f

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/man.rst

@@ -0,0 +1,13 @@
+:orphan:
+
+sn0int
+======
+
+todo
+
+.. toctree::
+    :maxdepth: 3
+    :glob:
+
+    usage
+    reference

docs/reference.rst

@@ -0,0 +1,467 @@
+Function reference
+==================
+
+clear_err
+---------
+
+Clear the last recorded error from the internal state. See also last_err_.
+
+.. code-block:: lua
+
+    if last_err() then
+        -- ignore this error
+        clear_err()
+    end
+
+db_add
+------
+
+Add an entity to the database or update it if it already exists. This function
+may fail or return ``nil``. See `db_add <database.html#db-add>`__ for details.
+
+.. code-block:: lua
+
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+
+db_select
+---------
+
+Checks if a target is in scope. If non-nil is returned, this entity is in
+scope. This function may fail. See `db_select <database.html#db-select>`__ for
+details.
+
+.. code-block:: lua
+
+    domain_id = db_select('domain', 'example.com')
+    if domain_id ~= nil then
+        -- do something
+    end
+
+db_update
+---------
+
+Update an entity in the database. This function may fail. See `db_update
+<database.html#db-update>`__ for details.
+
+.. code-block:: lua
+
+    db_update('ipaddr', arg, {
+        asn=lookup['asn'],
+        as_org=lookup['as_org'],
+    })
+
+dns
+---
+
+Resolve a dns record. In the dns response contains an error, ``x['error']`` is
+set and ``x['success']`` is ``nil``. Otherwise, ``x['success']`` contains a
+list of records. This function may fail.
+
+.. code-block:: lua
+
+    x = dns('example.com', 'A')
+    if last_err() then return end
+
+error
+-----
+
+Log an error to the terminal.
+
+.. code-block:: lua
+
+    error('ohai')
+
+asn_lookup
+----------
+
+Run an ASN lookup for a given ip address. The function returns ``asn`` and
+``as_org``. This function may fail.
+
+.. code-block:: lua
+
+    lookup = asn_lookup('1.1.1.1')
+    if last_err() then return end
+
+geoip_lookup
+------------
+
+Run a geoip lookup for a given ip address. The function returns:
+
+- continent
+- continent_code
+- country
+- country_code
+- city
+- latitude
+- longitude
+
+This function may fail.
+
+.. code-block:: lua
+
+    lookup = geoip_lookup('1.1.1.1')
+    if last_err() then return end
+
+html_select
+-----------
+
+Parses an html document and returns the first element that matches the css
+selector. The return value is a table with `text` being the inner text and
+`attrs` being a table of the elements attributes.
+
+.. code-block:: lua
+
+    csrf = html_select(html, 'input[name="csrf"]')
+    token = csrf["attrs"]["value"]
+
+html_select_list
+----------------
+
+Same as html_select_ but returns all matches instead of the first one.
+
+.. code-block:: lua
+
+    html_select_list(html, 'input[name="csrf"]')
+
+http_mksession
+--------------
+
+Create a session object. This is similar to ``requests.Session`` in
+python-requests and keeps track of cookies.
+
+.. code-block:: lua
+
+    session = http_mksession()
+
+http_request
+------------
+
+Prepares an http request. The first argument is the session reference and
+cookies from that session are copied into the request. After the request has
+been sent, the cookies from the response are copied back into the session.
+
+The next arguments are the ``method``, the ``url`` and additional options.
+Please note that you still need to specify an empty table ``{}`` even if no
+options are set. The following options are available:
+
+``query``
+  A map of query parameters that should be set on the url
+``headers``
+  A map of headers that should be set
+``basic_auth``
+  Configure the basic auth header with ``{"user, "password"}``
+``user_agent``
+  Overwrite the default user agent with a string
+``json``
+  The request body that should be json encoded
+``form``
+  The request body that should be form encoded
+``body``
+  The raw request body as string
+
+This function may fail.
+
+.. code-block:: lua
+
+    req = http_request(session, 'POST', 'https://httpbin.org/post', {
+        json={
+            user=user,
+            password=password,
+        }
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp["status"] ~= 200 then return "invalid status code" end
+
+http_send
+---------
+
+Send the request that has been built with http_request_. Returns a table with
+the following keys:
+
+``status``
+  The http status code
+``headers``
+  A table of headers
+``text``
+  The response body as string
+
+.. code-block:: lua
+
+    req = http_request(session, 'POST', 'https://httpbin.org/post', {
+        json={
+            user=user,
+            password=password,
+        }
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp["status"] ~= 200 then return "invalid status code" end
+
+info
+----
+
+Log an info to the terminal.
+
+.. code-block:: lua
+
+    info('ohai')
+
+json_decode
+-----------
+
+Decode a lua value from a json string.
+
+.. code-block:: lua
+
+    json_decode("{\"data\":{\"password\":\"fizz\",\"user\":\"bar\"},\"list\":[1,3,3,7]}")
+
+json_decode_stream
+------------------
+
+Very similar to json_decode_, but works with multiple json objects directly
+concatenated to each other or separated by newlines.
+
+.. code-block:: lua
+
+    json_decode_stream("{\"data\":1}{\"data\":2}")
+
+json_encode
+-----------
+
+Encode a datastructure into a string.
+
+.. code-block:: lua
+
+    x = json_encode({
+        some=1,
+        fancy={
+            data='structures',
+        }
+    })
+    print(x)
+
+last_err
+--------
+
+Returns infos about the last error we've observed, if any. Returns ``nil`` otherwise.
+
+.. code-block:: lua
+
+    if last_err() then
+        -- Something went wrong, abort
+        return
+    end
+
+pgp_pubkey
+----------
+
+Same as pgp_pubkey_armored_, but without the unarmor step.
+
+pgp_pubkey_armored
+------------------
+
+Extract uids out of a rfc 4880 pgp public key. This function may fail.
+
+.. code-block:: lua
+
+    key = pgp_pubkey_armored([===[
+    -----BEGIN PGP PUBLIC KEY BLOCK-----
+    Version: GnuPG v2
+
+    mQENBFu6q90BCADgD7Q9aH5683yt7hzPktDkAUNAZJHwYhUNeyGK43frPyDRWQmq
+    N+oXTfiYWLQN+d7KNBTnF9uwyBdaLM7SH44lLNYo8W09mVM2eK+wt19uf5HYNgAE
+    8la45QLo/ce9CQVe1a4oXNWq6l0FOY7M+wLe+G2wMwz8RXGgwd/qQp4/PB5YpUhx
+    nAnzClxvwymrL6BQXsRcKSMSD5bIzIv95n105CvW5Hql7JR9zgOR+gHqVOH8HBUc
+    ZxMumrTM6aKLgAhgM8Sn36gCFOfjlG1b1OFLZhUtgro/nnEOmAurRsCZy8M5h8QM
+    FpZChIH8kgHs90F/CCvGjMq3qvWcH8ZsPUizABEBAAG0NUhhbnMgQWNrZXIgKGV4
+    YW1wbGUgY29tbWVudCkgPGhhbnMuYWNrZXJAZXhhbXBsZS5jb20+iQFOBBMBCAA4
+    FiEEyzeO1eEwbB03hcqBM00IodGdlj8FAlu6q90CGwMFCwkIBwIGFQgJCgsCBBYC
+    AwECHgECF4AACgkQM00IodGdlj/AJQgAjmk+iP5b7Jt7+f+lU4Oprlf3f3DG/uh5
+    Ge6MjV7cvtxlhZJRD5hxGt9RwwnEp61TBSbrem288pM89ilQfTNe0wUr9OzwWzh/
+    8Ngl5iWnD2ah3Mpi5R1V/YMNf2cnwVjqNvfkRHdNc43pZOkC2GoiTUn0QY0UBpOW
+    ZMN3//ANi6ZtiK/L0IZQND/gKvOzu/4tfaJeBl26T3cVYj53p3G3jhlb92vVa8SR
+    uL3S3bzd1h5snDgU1uXHmNHGbhkEc4KUneQ0V9/bdZrg6OzFAfM1ghgfoId+YpQH
+    er9L26ISL3QF58wdEXfIdHYEmMlANjBMO2cUlQXgONuCgkMuY7GBmrkBDQRbuqvd
+    AQgA41jqCumCxYV0NdSYNnTSSDRyd69dOUYCAPT80iZ739s7KKJS9X9KVfGmDjfi
+    u2RcfR/KYj53HoyOm4Pm/+ONN8De4ktzXpIpJxGC+O8NBvd9vkboAS6qnCjK7KVE
+    r91ymxxVKp2dzZvVfpIjWVZR5i2EAvS5vw8UK4gL8ALH+S9leJFZrQWcgyoJOJzH
+    Rzr9pesX2HvdgcNG1O6QUArlsnsTnqpi/hu7tQa8tifBpWDeArOA23Y2DgeehdDF
+    lSU/8KD4J+AkFrWWlcTaMsvSChXQkCHEMRIcSOfXtdpX5KJSE7UBQdD1opm+mR79
+    VeHnuJAAVZZtUZmJA7pjdKykYQARAQABiQE2BBgBCAAgFiEEyzeO1eEwbB03hcqB
+    M00IodGdlj8FAlu6q90CGwwACgkQM00IodGdlj8bMAf+Lq3Qive4vcrCTT4IgvVj
+    arOACdcbtt5RhVBTimT19rDWNH+m+PfPjo3FSlBj5cm70KAXUS2LBFFxhakTZ/Mq
+    cQroWZpVbBxj4kipEVVJZFdUZQaDERJql0xYGOQrNMQ4JGqJ84BRrtOExjSqo41K
+    hAhNe+bwPGH9/Igiixc4tH07xa7TOy4MyJv/6gpbHy/lW1hqpCAgM5fT/im5/6QF
+    k0tED6vIuc54IWiOmwCnjZiQnJ8uCwEu+cuJ5Exwy9CNERLp5v0y4eG+0E+at9j/
+    macOg39qf09t53pTqe9dWv5NIi319TeBsKZ2lb0crrQjsbHqk0DAUwgQuoANqLku
+    vA==
+    =kRIv
+    -----END PGP PUBLIC KEY BLOCK-----
+    ]===])
+
+    if last_err() then return end
+    print(key)
+
+print
+-----
+
+Write something directly to the terminal.
+
+.. code-block:: lua
+
+    print({
+        some=1,
+        fancy={
+            data='structures',
+        }
+    })
+
+.. warning::
+   This function writes directly to the terminal and can interfere with other
+   terminal features. This function should be used during development only.
+
+psl_domain_from_dns_name
+------------------------
+
+Returns the parent domain according to the public suffix list. For
+``www.a.b.c.d.example.com`` this is going to be ``example.com``.
+
+.. code-block:: lua
+
+    domain = psl_domain_from_dns_name('www.a.b.c.d.example.com')
+    print(domain == 'example.com')
+
+regex_find
+----------
+
+Apply a regex to some text. Returns ``nil`` if the regex didn't match and the
+capture groups if it did.
+
+.. code-block:: lua
+
+    m = regex_find(".(.)", "abcdef")
+
+    if m == nil then
+        print('No captures')
+    end
+
+    print(m[1] == 'ab')
+    print(m[2] == 'b')
+
+regex_find_all
+--------------
+
+Same as regex_find_, but returns all matches.
+
+.. code-block:: lua
+
+    m = regex_find_all(".(.)", "abcdef")
+
+    print(m[1][1] == 'ab')
+    print(m[1][2] == 'b')
+    print(m[2][1] == 'cd')
+    print(m[2][2] == 'd')
+    print(m[3][1] == 'ef')
+    print(m[3][2] == 'f')
+
+sleep
+-----
+
+Pause the current program for the specified number of seconds. This is usually
+only used for debugging.
+
+.. code-block:: lua
+
+    sleep(1)
+
+status
+------
+
+Update the label of the progress indicator.
+
+.. code-block:: lua
+
+    status('ohai')
+
+url_join
+--------
+
+Join a relative link to an absolute link. If both links are absolute we just
+return the first one:
+
+.. code-block:: lua
+
+    x = url_join('https://example.com/x', '/foo')
+    print(x == 'https://example.com/foo')
+
+    x = url_join('https://example.com/x', 'https://github.com/')
+    print(x == 'https://github.com/')
+
+url_parse
+---------
+
+Parse a url into its components. The following components are returned:
+
+- scheme
+- host
+- port
+- path
+- query
+- fragment
+- params
+
+.. code-block:: lua
+
+    url = url_parse("https://example.com")
+    print(url['scheme'] == 'https')
+    print(url['host'] == 'example.com')
+    print(url['path'] == '/')
+
+utf8_decode
+-----------
+
+Decodes a list of bytes/numbers into a string. This function might fail.
+
+.. code-block:: lua
+
+    x = utf8_decode({65, 65, 65, 65})
+    if last_err() then return end
+    print(x == 'AAAA')
+
+x509_parse_pem
+--------------
+
+Parse a pem encoded certificate. This function might fail.
+
+.. code-block:: lua
+
+    x = x509_parse_pem([[-----BEGIN CERTIFICATE-----
+    MIID9DCCA3qgAwIBAgIQBWzetBRl/ycHFsBukRYuGTAKBggqhkjOPQQDAjBMMQsw
+    CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMSYwJAYDVQQDEx1EaWdp
+    Q2VydCBFQ0MgU2VjdXJlIFNlcnZlciBDQTAeFw0xODAzMzAwMDAwMDBaFw0yMDAz
+    MjUxMjAwMDBaMGwxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMN
+    U2FuIEZyYW5jaXNjbzEZMBcGA1UEChMQQ2xvdWRmbGFyZSwgSW5jLjEdMBsGA1UE
+    AwwUKi5jbG91ZGZsYXJlLWRucy5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC
+    AASyRQsxrFBjziHmfDQjGsXBU0WWl3oxh7vg6h2V9f8lBMp18PY/td9R6VvJPa20
+    AwVzIJI+dL6OSxviaIZEbmK7o4ICHDCCAhgwHwYDVR0jBBgwFoAUo53mH/naOU/A
+    buiRy5Wl2jHiCp8wHQYDVR0OBBYEFN+XTeVDs7BBp0LykM+Jf64SV4ThMGMGA1Ud
+    EQRcMFqCFCouY2xvdWRmbGFyZS1kbnMuY29thwQBAQEBhwQBAAABghJjbG91ZGZs
+    YXJlLWRucy5jb22HECYGRwBHAAAAAAAAAAAAERGHECYGRwBHAAAAAAAAAAAAEAEw
+    DgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBp
+    BgNVHR8EYjBgMC6gLKAqhihodHRwOi8vY3JsMy5kaWdpY2VydC5jb20vc3NjYS1l
+    Y2MtZzEuY3JsMC6gLKAqhihodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vc3NjYS1l
+    Y2MtZzEuY3JsMEwGA1UdIARFMEMwNwYJYIZIAYb9bAEBMCowKAYIKwYBBQUHAgEW
+    HGh0dHBzOi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwCAYGZ4EMAQICMHsGCCsGAQUF
+    BwEBBG8wbTAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQuY29tMEUG
+    CCsGAQUFBzAChjlodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGlnaUNlcnRF
+    Q0NTZWN1cmVTZXJ2ZXJDQS5jcnQwDAYDVR0TAQH/BAIwADAKBggqhkjOPQQDAgNo
+    ADBlAjEAjoyy2Ogh1i1/Kh9+psMc1OChlQIvQF6AkojZS8yliar6m8q5nqC3qe0h
+    HR0fExwLAjAueWRnHX4QJ9loqMhsPk3NB0Cs0mStsNDNG6/DpCYw7XmjoG3y1LS7
+    ZkZZmqNn2Q8=
+    -----END CERTIFICATE-----
+    ]])
+    if last_err() then return end
+    print(x)

docs/scripting.rst

@@ -0,0 +1,183 @@
+Scripting
+=========
+
+Scripting is the core feature in sn0int. It's not strictly required, but if you
+want to write your own modules, this section is for you.
+
+Write your first module
+-----------------------
+
+It's highly recommended to use a VCS for development, so let's start by setting
+that up. We're going to assume you store your repos in ``~/repos`` but you're
+free to change that to something else::
+
+    $ git init ~/repos/sn0int-modules
+    $ cd ~/repos/sn0int-modules
+    $ ln -s "$PWD" ~/.local/share/sn0int/modules/$YOUR_GITHUB_NAME
+
+Every module we're adding to ``~/repos/sn0int-modules`` is now going to be
+picked up by sn0int.
+
+Let's add our first module by opening ``~/repos/sn0int-modules/first.lua``.
+There's a bit of boilerplate that every module needs to load successfully:
+
+.. code-block:: lua
+
+    -- Description: ohai wurld
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
+    function run(arg)
+        -- TODO: do something here
+    end
+
+``Description`` (mandatory)
+  This should be a short text that describes what your module is doing.
+
+``Version`` (mandatory)
+  Every module requires a semver_ version. You can just set it to ``0.1.0``
+  during development, but you need to increase it every time you publish your
+  module. If you don't care about that one, just keep increasing ``0.X.0``.
+
+.. _semver: https://semver.org/
+
+``Source`` (mandatory)
+  This is going to specify what kind of entities we're interested in. If we
+  specify ``domains`` our module is going to be called with all domains that
+  are targeted.
+
+  - ``domains``
+  - ``subdomains``
+  - ``ipaddrs``
+  - ``urls``
+  - ``emails``
+
+``License`` (mandatory)
+  This is somewhat special. We require that every module is licensed under an
+  open source license. Pick one of the following licenses.
+
+  - ``MIT`` -  https://opensource.org/licenses/MIT
+  - ``GPL-3.0`` - https://opensource.org/licenses/gpl-license
+  - ``LGPL-3.0`` - https://opensource.org/licenses/lgpl-license
+  - ``BSD-2-Clause`` - https://opensource.org/licenses/BSD-2-Clause
+  - ``BSD-3-Clause`` - https://opensource.org/licenses/BSD-3-Clause
+  - ``WTFPL`` - https://spdx.org/licenses/WTFPL.html
+
+``function run(arg)`` (mandatory)
+  This is where the actual magic of our module happens. Our function is going
+  to be called in a loop for each entity that is targeted by the user.
+
+Let's continue. For the sake of an hello world we're going to take some
+``domains``, check if a ``www`` subdomain exists and if it does, add it to the
+database.
+
+.. code-block:: lua
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+        print(subdomain)
+    end
+
+Combined with the header we wrote previously we can already execute this
+module. Make sure you've added a domain to scope with ``add domain
+example.com``, save your file and run it like this::
+
+    sn0int run -f ./first.lua
+
+We should see some output by our print function.
+
+.. note::
+   ``print`` is useful for development but must be removed before publishing.
+
+Next, we want to actually resolve that name, we're going to use the ``dns``
+function for that. This function takes a name and a query type and returns a
+result. Note that this function might fail, in which case we want to abort our
+function. We do that by checking if the return value of ``last_err()`` is
+truth-y.
+
+.. code-block:: lua
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+
+        records = dns(subdomain, 'A')
+        if last_err() then return end
+
+        print(records)
+    end
+
+If you run your module again you're going to see some output, either
+``{"success": somedata}`` or ``{"error": "NX"}``. We decide if ``success`` is
+not ``nil``, we add the subdomain to our scope and set it to resolvable:
+
+.. code-block:: lua
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+
+        records = dns(subdomain, 'A')
+        if last_err() then return end
+
+        if records['success'] ~= nil then
+            db_add('subdomain', arg, {
+                domain_id=arg['id'],
+                value=subdomain,
+                resolvable=true,
+            })
+        end
+    end
+
+.. hint::
+   See the database section to understand how the database works in detail.
+
+If we execute our module one more time it's going to log that it discovered a
+subdomain, if it doesn't, try adding more domains to scope. Note that this only
+happens the first time. Modules that don't discover anything or don't discover
+anything new exit silently.
+
+After putting everything together, our final module looks like this:
+
+.. code-block:: lua
+
+    -- Description: ohai wurld
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+
+        records = dns(subdomain, 'A')
+        if last_err() then return end
+
+        if records['success'] ~= nil then
+            db_add('subdomain', arg, {
+                domain_id=arg['id'],
+                value=subdomain,
+                resolvable=true,
+            })
+        end
+    end
+
+There's still some room for improvement, for example, since we already resolved
+that record, we could also add the ip address to the scope and link it to the
+subdomain we added.
+
+Publish your module
+-------------------
+
+The public registry uses github usernames to namespace the registry. This means
+you need to authenticate to the registry using your github username. This can
+be done using::
+
+    sn0int login
+
+sn0int is going to open a new tab in your browser, if you are already signed
+into your github account you only need to confirm an authorization request. The
+application doesn't need any of your data, so it's only asking you to confirm
+your identity.
+
+Afterwards publish your module with::
+
+    sn0int publish ./first.lua

docs/usage.rst

@@ -0,0 +1,258 @@
+Running your first investigation
+================================
+
+This page is going to guide you through the process of setting up your
+environment and running your first investigation.
+
+Installing the default modules
+------------------------------
+
+By default, sn0int doesn't have any modules installed. If you start up sn0int
+it's going to download some files that it needs and then suggests to install a
+number of recommended modules::
+
+    $ sn0int
+
+                       ___/           .
+         ____ , __   .'  /\ ` , __   _/_
+        (     |'  `. |  / | | |'  `.  |
+        `--.  |    | |,'  | | |    |  |
+       \___.' /    | /`---' / /    |  \__/
+
+            osint | recon | security
+          irc.hackint.org:6697/#sn0int
+
+    [+] Connecting to database
+    [+] Downloading public suffix list
+    [+] Downloading "GeoLite2-City.mmdb"
+    [+] Downloading "GeoLite2-ASN.mmdb"
+    [+] Loaded 0 modules
+    [*] No modules found, run quickstart to install default modules
+    [sn0int][default] >
+
+Typing ``quickstart`` is going to get you a fair number of featured modules::
+
+    [sn0int][default] > quickstart
+    [+] Installing kpcyrd/asn
+    [+] Installing kpcyrd/ctlogs
+    [+] Installing kpcyrd/dns-resolve
+    [+] Installing kpcyrd/geoip
+    [+] Installing kpcyrd/hackertarget-subdomains
+    [+] Installing kpcyrd/otx-subdomains
+    [+] Installing kpcyrd/passive-spider
+    [+] Installing kpcyrd/pgp-keyserver
+    [+] Installing kpcyrd/threatminer-ipaddr
+    [+] Installing kpcyrd/threatminer-subdomains
+    [+] Installing kpcyrd/url-scan
+    [+] Installing kpcyrd/waybackurls
+    [+] Loaded 12 modules
+    [sn0int][default] >
+
+Adding something to scope
+-------------------------
+
+You probably want to separate your investigations so you should select a
+workspace where your results should go::
+
+    [sn0int][default] > workspace demo
+    [+] Connecting to database
+    [sn0int][demo] >
+
+Next, we have to start somewhere and add the first entity to our scope::
+
+    [sn0int][demo] > add domain
+    Domain: example.com
+    [sn0int][demo] >
+
+.. note::
+   There is a concept of a domain vs a subdomain. We are referring to a domain
+   as everything that is a subdomain of a `public suffix`_. For example, .com
+   is a public suffix, which makes example.com a domain in sn0int terms. Every
+   subdomain of that, like www.example.com, is referred to as a subdomain.
+
+   Note that example.com can be added as a subdomain as well since it can hold
+   records. In that case, example.com is both the name of the dns zone, while
+   also being an entity in that zone.
+
+.. _public suffix: https://publicsuffix.org/
+
+You can confirm this by running a select on the domains we now have::
+
+    [sn0int][demo] > select domains
+    #1, "example.com"
+    [sn0int][demo] >
+
+Something we don't need right now, but is going to be useful later on is the
+ability to filter your entities::
+
+    [sn0int][demo] > select domains where id=1
+    #1, "example.com"
+    [sn0int][demo] >
+    [sn0int][demo] > select domains where value like %.com
+    #1, "example.com"
+    [sn0int][demo] >
+    [sn0int][demo] > select domains where ( value like e% and value like %m ) or false
+    #1, "example.com"
+    [sn0int][demo] >
+
+.. note::
+   Almost all entities have a ``value`` column that holds the primary value of
+   the entity.
+
+Running a module
+----------------
+
+Now that we have something to get started with, we can run our first module.
+First lets list all modules we have::
+
+    [sn0int][demo] > mod list
+    kpcyrd/asn (0.1.0)
+        Run a asn lookup for an ip address
+    kpcyrd/ctlogs (0.1.0)
+        Query certificate transparency logs to discover subdomains
+    kpcyrd/dns-resolve (0.1.0)
+        Query subdomains to discovery ip addresses and verify the record is visible
+    kpcyrd/geoip (0.1.0)
+        Run a geoip lookup for an ip address
+    kpcyrd/hackertarget-subdomains (0.1.0)
+        Query hackertarget for subdomains of a domain
+    kpcyrd/otx-subdomains (0.1.0)
+        Query alienvault otx passive dns for subdomains of a domain
+    kpcyrd/passive-spider (0.1.0)
+        Scrape known http responses for urls
+    kpcyrd/pgp-keyserver (0.1.0)
+        Query pgp keyserver for email addresses
+    kpcyrd/threatminer-ipaddr (0.1.0)
+        Query ThreatMiner passive dns for subdomains of an ip address
+    kpcyrd/threatminer-subdomains (0.1.0)
+        Query ThreatMiner passive dns for subdomains of a domain
+    kpcyrd/url-scan (0.1.0)
+        Scan subdomains for websites
+    kpcyrd/waybackurls (0.1.0)
+        Discover subdomains from wayback machine
+    [sn0int][demo] >
+
+Let's start by querying certificate transparency logs::
+
+    [sn0int][demo] > use ctlogs
+    [sn0int][demo][kpcyrd/ctlogs] > run
+    [*] "example.com"                                     : Subdomain: "www.example.com"
+    [*] "example.com"                                     : Subdomain: "m.example.com"
+    [*] "example.com"                                     : Subdomain: "dev.example.com"
+    [*] "example.com"                                     : Subdomain: "products.example.com"
+    [*] "example.com"                                     : Subdomain: "support.example.com"
+    [+] Finished kpcyrd/ctlogs
+    [sn0int][demo][kpcyrd/ctlogs] >
+
+Looks like we've discovered some subdomains here. It might be tempting to throw
+some of them in a browser but hold on, there's a more efficient way to approach
+this.
+
+.. hint::
+   You can run the modules concurrently with ``run -j 8``.
+
+Running followup modules on the results
+---------------------------------------
+
+A lot of time has been spent on the database part. While it sort of feels like
+a no-sql database we are actually enforcing a schema for a reason instead of
+just using generic dictionaries and calling it a day.
+
+It's crucial that entities created by one module can be picked up by another
+module, like LEGOs. Let's continue with a module to query the dns records::
+
+    [sn0int][demo][kpcyrd/ctlogs] > use dns-resolve
+    [sn0int][demo][kpcyrd/dns-resolve] > run
+    [*] "www.example.com"                                 : Updating "www.example.com" (resolvable => true)
+    [*] "www.example.com"                                 : IpAddr: 93.184.216.34
+    [*] "www.example.com"                                 : "www.example.com" -> 93.184.216.34
+    [*] "m.example.com"                                   : Updating "m.example.com" (resolvable => false)
+    [*] "dev.example.com"                                 : Updating "dev.example.com" (resolvable => false)
+    [*] "products.example.com"                            : Updating "products.example.com" (resolvable => false)
+    [*] "support.example.com"                             : Updating "support.example.com" (resolvable => false)
+    [+] Finished kpcyrd/dns-resolve
+    [sn0int][demo][kpcyrd/dns-resolve] >
+
+.. TODO: mention https://github.com/kpcyrd/sn0int/issues/27
+
+Two things happened here: We've discovered some IP addresses and added them to
+scope, and we also updated our subdomain entities with new information, since
+we now know which of them are resolvable and which aren't.
+
+Let's run the next module, which is actually going to check for websites on
+them, but let's only target subdomains that we know are resolvable::
+
+    [sn0int][demo][kpcyrd/dns-resolve] > use url-scan
+    [sn0int][demo][kpcyrd/url-scan] > target
+    #1, "www.example.com"
+        93.184.216.34
+    #2, "m.example.com"
+    #3, "dev.example.com"
+    #4, "products.example.com"
+    #5, "support.example.com"
+    [sn0int][demo][kpcyrd/url-scan] > target where resolvable
+    [+] 1 entities selected
+    [sn0int][demo][kpcyrd/url-scan] > target
+    #1, "www.example.com"
+        93.184.216.34
+    [sn0int][demo][kpcyrd/url-scan] >
+
+We can both preview and limit the targets that are going to be passed to the
+module with the target command. Once we are satisfied with our selection we can
+run this module::
+
+    [sn0int][demo][kpcyrd/url-scan] > run
+    [*] "www.example.com"                                 : Url: "http://www.example.com/" (200)
+    [*] "www.example.com"                                 : Url: "https://www.example.com/" (200)
+    [+] Finished kpcyrd/url-scan
+    [sn0int][demo][kpcyrd/url-scan] >
+
+We've now probed both port 80 and port 443 for each subdomain and found two
+http responses this way. If you want a list of urls you may want to visit in
+your browser can now query them::
+
+    [sn0int][demo][kpcyrd/url-scan] > select urls
+    #1, "http://www.example.com/" (200)
+    #2, "https://www.example.com/" (200)
+    [sn0int][demo][kpcyrd/url-scan] >
+
+Unscoping entities
+------------------
+
+Something you are going to run into is that modules are too greedy and add
+things to the scope we are not interested in. You can delete them using the
+delete command, but those are likely picked up by a module again.
+
+What you can do instead is setting a flag on an entity that removes it from
+our scope. This is done using the noscope command::
+
+    [sn0int][demo] > use ctlogs
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    [sn0int][demo][kpcyrd/ctlogs] > add domain
+    Domain: google.com
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    #2, "google.com"
+    [sn0int][demo][kpcyrd/ctlogs] > noscope domains where value=google.com
+    [+] Updated 1 rows
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    [sn0int][demo][kpcyrd/ctlogs] >
+
+Entities that are unscoped are automatically ignored by all modules.
+
+You can reverse this using the scope command::
+
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    [sn0int][demo][kpcyrd/ctlogs] > scope domains where true
+    [+] Updated 2 rows
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    #2, "google.com"
+    [sn0int][demo][kpcyrd/ctlogs] >
+
+.. hint::
+   All entities have this field, you can refer to it in queries using
+   ``unscoped=1``.

examples/asn.rs

@@ -0,0 +1,32 @@
+extern crate sn0int;
+extern crate env_logger;
+extern crate maxminddb;
+
+use std::env;
+use sn0int::errors::*;
+use sn0int::geoip::{AsnDB, Maxmind};
+
+
+fn run() -> Result<()> {
+    let asndb = AsnDB::open_or_download()?;
+
+    for arg in env::args().skip(1) {
+        let ip = arg.parse()?;
+        let asn = asndb.lookup(ip)?;
+        println!("{:#?}", asn);
+    }
+
+    Ok(())
+}
+
+fn main() {
+    env_logger::init();
+
+    if let Err(err) = run() {
+        eprintln!("Error: {}", err);
+        for cause in err.iter_chain().skip(1) {
+            eprintln!("Because: {}", cause);
+        }
+        std::process::exit(1);
+    }
+}

examples/geoip.rs

@@ -0,0 +1,31 @@
+extern crate sn0int;
+extern crate env_logger;
+
+use std::env;
+use sn0int::errors::*;
+use sn0int::geoip::{GeoIP, Maxmind};
+
+
+fn run() -> Result<()> {
+    let geoip = GeoIP::open_or_download()?;
+
+    for arg in env::args().skip(1) {
+        let ip = arg.parse()?;
+        let lookup = geoip.lookup(ip)?;
+        println!("{:#?}", lookup);
+    }
+
+    Ok(())
+}
+
+fn main() {
+    env_logger::init();
+
+    if let Err(err) = run() {
+        eprintln!("Error: {}", err);
+        for cause in err.iter_chain().skip(1) {
+            eprintln!("Because: {}", cause);
+        }
+        std::process::exit(1);
+    }
+}

examples/maxmind-dl.rs

@@ -0,0 +1,50 @@
+extern crate sn0int;
+extern crate env_logger;
+extern crate chrootable_https;
+#[macro_use] extern crate log;
+
+// workaround for rustc 1.29.2 support
+#[cfg(not(target_os = "openbsd"))]
+extern crate structopt;
+#[cfg(target_os = "openbsd")]
+#[macro_use] extern crate structopt;
+
+use sn0int::errors::*;
+use sn0int::geoip::{GeoIP, Maxmind};
+use sn0int::paths;
+use std::fs;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub struct Args {
+    url: String,
+    filter: String,
+    target: String,
+    #[structopt(short="e", long="extract-only")]
+    extract_only: bool,
+}
+
+fn run() -> Result<()> {
+    let args = Args::from_args();
+    debug!("{:?}", args);
+    let path = paths::cache_dir()?.join(&args.target);
+    if args.extract_only {
+        let body = fs::read(&args.url)?;
+        sn0int::archive::extract(&mut &body[..], &args.filter, path)?;
+    } else {
+        GeoIP::download(path, &args.filter, &args.url)?;
+    }
+    Ok(())
+}
+
+fn main() {
+    env_logger::init();
+
+    if let Err(err) = run() {
+        eprintln!("Error: {}", err);
+        for cause in err.iter_chain().skip(1) {
+            eprintln!("Because: {}", cause);
+        }
+        std::process::exit(1);
+    }
+}

examples/stacked-spinners.rs

@@ -0,0 +1,25 @@
+extern crate sn0int;
+
+use sn0int::term::StackedSpinners;
+use std::thread;
+use std::time::Duration;
+
+fn main() {
+    let mut stack = StackedSpinners::new();
+    stack.add("1".into(), String::from("spinner1"));
+    stack.add("2".into(), String::from("spinner2"));
+    stack.add("3".into(), String::from("spinner3"));
+
+    for x in 1..=3 {
+        for _ in 0..50 {
+            thread::sleep(Duration::from_millis(100));
+            stack.tick();
+        }
+        // stack.log("ohai");
+        stack.remove(&x.to_string());
+    }
+
+    stack.clear();
+
+    // stack.finish("Done".to_string());
+}

migrations/2018-09-12-102647_create_domains/up.sql

@@ -24,7 +24,7 @@ CREATE TABLE subdomain_ipaddrs (
     id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
     subdomain_id INTEGER NOT NULL,
     ip_addr_id INTEGER NOT NULL,
-    FOREIGN KEY(subdomain_id) REFERENCES domains(id),
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
     FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id),
     CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
 );
@@ -35,6 +35,6 @@ CREATE TABLE urls (
     value VARCHAR NOT NULL,
     status INTEGER,
     body BLOB,
-    FOREIGN KEY(subdomain_id) REFERENCES domains(id),
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
     CONSTRAINT url_unique UNIQUE (value)
 );

migrations/2018-10-12-051052_emails/down.sql

@@ -0,0 +1 @@
+DROP TABLE emails;

migrations/2018-10-12-051052_emails/up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    CONSTRAINT email_unique UNIQUE (value)
+);

migrations/2018-10-15-061351_scope/down.sql

@@ -0,0 +1,90 @@
+PRAGMA foreign_keys=off;
+
+-- domains
+
+ALTER TABLE domains RENAME TO _domains_old;
+
+CREATE TABLE domains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    CONSTRAINT domain_unique UNIQUE (value)
+);
+
+INSERT INTO domains (id, value)
+  SELECT id, value
+  FROM _domains_old;
+
+DROP TABLE _domains_old;
+
+-- subdomains
+
+ALTER TABLE subdomains RENAME TO _subdomains_old;
+
+CREATE TABLE subdomains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    domain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    FOREIGN KEY(domain_id) REFERENCES domains(id),
+    CONSTRAINT subdomain_unique UNIQUE (value)
+);
+
+INSERT INTO subdomains (id, domain_id, value)
+  SELECT id, domain_id, value
+  FROM _subdomains_old;
+
+DROP TABLE _subdomains_old;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value)
+  SELECT id, family, value
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+-- urls
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body)
+  SELECT id, subdomain_id, value, status, body
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+-- emails
+
+ALTER TABLE emails RENAME TO _emails_old;
+
+CREATE TABLE emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    CONSTRAINT domain_unique UNIQUE (value)
+);
+
+INSERT INTO emails (id, value)
+  SELECT id, value
+  FROM _emails_old;
+
+DROP TABLE _emails_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-10-15-061351_scope/up.sql

@@ -0,0 +1,5 @@
+ALTER TABLE domains ADD COLUMN unscoped BOOLEAN DEFAULT 0 NOT NULL;
+ALTER TABLE subdomains ADD COLUMN unscoped BOOLEAN DEFAULT 0 NOT NULL;
+ALTER TABLE ipaddrs ADD COLUMN unscoped BOOLEAN DEFAULT 0 NOT NULL;
+ALTER TABLE urls ADD COLUMN unscoped BOOLEAN DEFAULT 0 NOT NULL;
+ALTER TABLE emails ADD COLUMN unscoped BOOLEAN DEFAULT 0 NOT NULL;

migrations/2018-10-17-112220_verifiable/down.sql

@@ -0,0 +1,60 @@
+PRAGMA foreign_keys=off;
+
+-- subdomains
+
+ALTER TABLE subdomains RENAME TO _subdomains_old;
+
+CREATE TABLE subdomains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    domain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    FOREIGN KEY(domain_id) REFERENCES domains(id),
+    CONSTRAINT subdomain_unique UNIQUE (value)
+);
+
+INSERT INTO subdomains (id, domain_id, value, unscoped)
+  SELECT id, domain_id, value, unscoped
+  FROM _subdomains_old;
+
+DROP TABLE _subdomains_old;
+
+-- emails
+
+ALTER TABLE emails RENAME TO _emails_old;
+
+CREATE TABLE emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT emails_unique UNIQUE (value)
+);
+
+INSERT INTO emails (id, value, unscoped)
+  SELECT id, value, unscoped
+  FROM _emails_old;
+
+DROP TABLE _emails_old;
+
+-- urls
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped)
+  SELECT id, subdomain_id, value, status, body, unscoped
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-10-17-112220_verifiable/up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE subdomains ADD COLUMN resolvable BOOLEAN;
+ALTER TABLE emails ADD COLUMN valid BOOLEAN;
+ALTER TABLE urls ADD COLUMN online BOOLEAN;

migrations/2018-10-22-141819_geoip/down.sql

@@ -0,0 +1,21 @@
+PRAGMA foreign_keys=off;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value, unscoped)
+  SELECT id, family, value, unscoped
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-10-22-141819_geoip/up.sql

@@ -0,0 +1,7 @@
+ALTER TABLE ipaddrs ADD COLUMN continent VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN continent_code VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN country VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN country_code VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN city VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN latitude FLOAT;
+ALTER TABLE ipaddrs ADD COLUMN longitude FLOAT;

migrations/2018-10-24-131856_asn/down.sql

@@ -0,0 +1,28 @@
+PRAGMA foreign_keys=off;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude)
+  SELECT id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-10-24-131856_asn/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ipaddrs ADD COLUMN asn INTEGER;
+ALTER TABLE ipaddrs ADD COLUMN as_org VARCHAR;

migrations/2018-10-26-001932_cascade-delete/down.sql

@@ -0,0 +1,126 @@
+PRAGMA foreign_keys=off;
+
+-- domains
+
+ALTER TABLE domains RENAME TO _domains_old;
+
+CREATE TABLE domains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT domain_unique UNIQUE (value)
+);
+
+INSERT INTO domains (id, value, unscoped)
+    SELECT id, value, unscoped
+    FROM _domains_old;
+
+DROP TABLE _domains_old;
+
+-- subdomains
+
+ALTER TABLE subdomains RENAME TO _subdomains_old;
+
+CREATE TABLE subdomains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    domain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    resolvable BOOLEAN,
+    FOREIGN KEY(domain_id) REFERENCES domains(id),
+    CONSTRAINT subdomain_unique UNIQUE (value)
+);
+
+INSERT INTO subdomains (id, domain_id, value, unscoped, resolvable)
+    SELECT id, domain_id, value, unscoped, resolvable
+    FROM _subdomains_old;
+
+DROP TABLE _subdomains_old;
+
+-- subdomain_ipaddrs
+
+ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
+
+CREATE TABLE subdomain_ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id),
+    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
+);
+
+INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
+    SELECT id, subdomain_id, ip_addr_id
+    FROM _subdomain_ipaddrs_old;
+
+DROP TABLE _subdomain_ipaddrs_old;
+
+-- urls
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online)
+    SELECT id, subdomain_id, value, status, body, unscoped, online
+    FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+-- emails
+
+ALTER TABLE emails RENAME TO _emails_old;
+
+CREATE TABLE emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO emails (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM _emails_old;
+
+DROP TABLE _emails_old;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    asn INTEGER,
+    as_org VARCHAR,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude, asn, as_org)
+  SELECT id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude, asn, as_org
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-10-26-001932_cascade-delete/up.sql

@@ -0,0 +1,126 @@
+PRAGMA foreign_keys=off;
+
+-- domains
+
+ALTER TABLE domains RENAME TO _domains_old;
+
+CREATE TABLE domains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT domain_unique UNIQUE (value)
+);
+
+INSERT INTO domains (id, value, unscoped)
+    SELECT id, value, unscoped
+    FROM _domains_old;
+
+DROP TABLE _domains_old;
+
+-- subdomains
+
+ALTER TABLE subdomains RENAME TO _subdomains_old;
+
+CREATE TABLE subdomains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    domain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    resolvable BOOLEAN,
+    FOREIGN KEY(domain_id) REFERENCES domains(id) ON DELETE CASCADE,
+    CONSTRAINT subdomain_unique UNIQUE (value)
+);
+
+INSERT INTO subdomains (id, domain_id, value, unscoped, resolvable)
+    SELECT id, domain_id, value, unscoped, resolvable
+    FROM _subdomains_old;
+
+DROP TABLE _subdomains_old;
+
+-- subdomain_ipaddrs
+
+ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
+
+CREATE TABLE subdomain_ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
+);
+
+INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
+    SELECT id, subdomain_id, ip_addr_id
+    FROM _subdomain_ipaddrs_old;
+
+DROP TABLE _subdomain_ipaddrs_old;
+
+-- urls
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online)
+    SELECT id, subdomain_id, value, status, body, unscoped, online
+    FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+-- emails
+
+ALTER TABLE emails RENAME TO _emails_old;
+
+CREATE TABLE emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO emails (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM _emails_old;
+
+DROP TABLE _emails_old;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    asn INTEGER,
+    as_org VARCHAR,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude, asn, as_org)
+  SELECT id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude, asn, as_org
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-11-03-230746_extend-urls/down.sql

@@ -0,0 +1,25 @@
+PRAGMA foreign_keys=off;
+
+-- urls
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online)
+    SELECT id, subdomain_id, value, status, body, unscoped, online
+    FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-11-03-230746_extend-urls/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE urls ADD COLUMN title VARCHAR;
+ALTER TABLE urls ADD COLUMN redirect VARCHAR;

modules/dev/54267.lua

@@ -1,12 +0,0 @@
--- Description: Reproduce a rust stdlib panic
--- Version: 0.1.0
--- Source: domains
-
-function run()
-    --[[
-    See for details:
-    https://github.com/rust-lang/rust/issues/54267
-    https://github.com/rust-lang/rust/issues/39364
-    ]]--
-    sleep(10)
-end

modules/dev/asn.lua

@@ -0,0 +1,16 @@
+-- Description: Run a asn lookup for an ip address
+-- Version: 0.1.0
+-- Source: ipaddrs
+-- License: GPL-3.0
+
+function run(arg)
+    lookup = asn_lookup(arg['value'])
+    if last_err() then return end
+
+    if arg['asn'] ~= lookup['asn'] or arg['as_org'] ~= lookup['as_org'] then
+        db_update('ipaddr', arg, {
+            asn=lookup['asn'],
+            as_org=lookup['as_org'],
+        })
+    end
+end

modules/dev/ctlogs.lua

@@ -1,11 +1,11 @@
 -- Description: Query certificate transparency logs to discover subdomains
 -- Version: 0.1.0
 -- Source: domains
+-- License: GPL-3.0
 
 function run(arg)
     session = http_mksession()
 
-    -- TODO: example.com needs to be dynamic
     req = http_request(session, 'GET', 'https://crt.sh/', {
         query={
             q='%.' .. arg['value'],

modules/dev/dns-resolve.lua

@@ -1,13 +1,22 @@
 -- Description: Query subdomains to discovery ip addresses and verify the record is visible
 -- Version: 0.1.0
 -- Source: subdomains
+-- License: GPL-3.0
 
 function run(arg)
     records = dns(arg['value'], 'A')
     if last_err() then return end
 
-    if records['success'] == nil then
-        -- TODO: consider marking the subdomain is inaccessible
+    -- update subdomain
+    resolvable = records['success'] ~= nil
+    if arg['resolvable'] ~= resolvable then
+        -- TODO: pass arg to function as well
+        db_update('subdomain', arg, {
+            resolvable=resolvable
+        })
+    end
+
+    if not resolvable then
         return
     end
 

modules/dev/geoip.lua

@@ -0,0 +1,10 @@
+-- Description: Run a geoip lookup for an ip address
+-- Version: 0.1.0
+-- Source: ipaddrs
+-- License: GPL-3.0
+
+function run(arg)
+    lookup = geoip_lookup(arg['value'])
+    if last_err() then return end
+    db_update('ipaddr', arg, lookup)
+end

modules/dev/hackertarget-subdomains.lua

@@ -0,0 +1,30 @@
+-- Description: Query hackertarget for subdomains of a domain
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run(arg)
+    session = http_mksession()
+
+    req = http_request(session, 'GET', 'https://api.hackertarget.com/hostsearch/', {
+        query={
+            q=arg['value']
+        }
+    })
+
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    m = regex_find_all("([^,]+),.+\\n?", resp['text'])
+
+    i = 1
+    while i <= #m do
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=m[i][2]
+        })
+
+        i = i+1
+    end
+end

modules/dev/otx-subdomains.lua

@@ -0,0 +1,32 @@
+-- Description: Query alienvault otx passive dns for subdomains of a domain
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run(arg)
+    session = http_mksession()
+
+    url = 'https://otx.alienvault.com/api/v1/indicators/domain/' .. arg['value'] .. '/passive_dns'
+
+    req = http_request(session, 'GET', url, {})
+
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    o = json_decode(resp['text'])
+    if last_err() then return end
+    o = o['passive_dns']
+
+    i = 0
+    while o[i] do
+        x = o[i]
+
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=x['hostname'],
+        })
+
+        i = i+1
+    end
+end

modules/dev/passive-spider.lua

@@ -0,0 +1,63 @@
+-- Description: Scrape known http responses for urls
+-- Version: 0.1.0
+-- Source: urls
+-- License: GPL-3.0
+
+function entry(parent, href)
+    -- TODO: parse mailto:foo@example.com?subject=asdf
+    -- TODO: parse tel:+4912345
+    -- TODO: allow discovering 3rd-party domains
+    -- TODO: maybe record urls as well
+
+    local psl, parts, url, host
+
+    if href == nil then
+        return
+    end
+
+    url = url_join(parent, href)
+    if last_err() then return clear_err() end
+    if url:match('^https?://') == nil then
+        return
+    end
+
+    parts = url_parse(url)
+    if last_err() then return end
+    host = parts['host']
+    psl = psl_domain_from_dns_name(host)
+
+
+    domain_id = db_select('domain', psl)
+    if domain_id ~= nil then
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=host,
+        })
+    end
+end
+
+function run(arg)
+    if arg['body'] == nil or #arg['body'] == 0 then
+        return
+    end
+
+    body = utf8_decode(arg['body'])
+    if last_err() then return end
+
+    links = html_select_list(body, 'a')
+    if last_err() then return end
+
+    if #links == 0 then
+        return
+    end
+
+    -- process html links
+    i = 1
+    while i <= #links do
+        href = links[i]['attrs']['href']
+
+        entry(arg['value'], href)
+
+        i = i+1
+    end
+end

modules/dev/pgp-keyserver.lua

@@ -0,0 +1,58 @@
+-- Description: Query pgp keyserver for email addresses
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run(arg)
+    session = http_mksession()
+
+    --lookup_url = 'https://pgp.mit.edu/pks/lookup'
+    lookup_url = 'https://sks-keyservers.net/pks/lookup'
+
+    req = http_request(session, 'GET', lookup_url, {
+        query={
+            search=arg['value'],
+        }
+    })
+
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    links = html_select_list(resp['text'], 'a')
+    i = 1
+    while i <= #links do
+        href = links[i]['attrs']['href']
+
+        if href:find('/pks/lookup%?op=get&search=') == 1 then
+            url = url_join(lookup_url, href)
+
+            req = http_request(session, 'GET', url, {})
+
+            resp = http_send(req)
+            -- TODO: do not abort script if one attempt fails
+            if last_err() then return end
+            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+            pubkey = pgp_pubkey_armored(resp['text'])
+
+            -- print(pubkey)
+
+            -- TODO: ensure at least one email matches our target domain
+            if pubkey['uids'] then
+                j = 1
+                while j <= #pubkey['uids'] do
+                    m = regex_find("<([^< ]+@[^< ]+)>$", pubkey['uids'][j])
+                    if m then
+                        db_add('email', {
+                            value=m[2],
+                        })
+                    end
+                    j = j+1
+                end
+            end
+        end
+
+        i = i+1
+    end
+end

modules/dev/threatminer-ipaddr.lua

@@ -0,0 +1,52 @@
+-- Description: Query ThreatMiner passive dns for subdomains of an ip address
+-- Version: 0.1.0
+-- Source: ipaddrs
+-- License: GPL-3.0
+
+function run(arg)
+    session = http_mksession()
+
+    -- TODO: add option to filter old entries based on last_seen
+
+    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/host.php', {
+        query={
+            rt='2',
+            q=arg['value']
+        }
+    })
+
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    o = json_decode(resp['text'])
+    if last_err() then return end
+    o = o['results']
+
+    i = 0
+    while o[i] do
+        x = o[i]
+
+        domain = psl_domain_from_dns_name(x['domain'])
+        -- TODO: if this fails, skip this entry instead
+        if last_err() then return end
+
+        domain_id = db_add('domain', {
+            value=domain,
+        })
+
+        if domain_id ~= nil then
+            subdomain_id = db_add('subdomain', {
+                domain_id=domain_id,
+                value=x['domain'],
+            })
+
+            db_add('subdomain-ipaddr', {
+                subdomain_id=subdomain_id,
+                ip_addr_id=arg['id'],
+            })
+        end
+
+        i = i+1
+    end
+end

modules/dev/threatminer-subdomains.lua

@@ -0,0 +1,35 @@
+-- Description: Query ThreatMiner passive dns for subdomains of a domain
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run(arg)
+    session = http_mksession()
+
+    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/domain.php', {
+        query={
+            rt='5',
+            q=arg['value']
+        }
+    })
+
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    o = json_decode(resp['text'])
+    if last_err() then return end
+    o = o['results']
+
+    i = 0
+    while o[i] do
+        x = o[i]
+
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=x,
+        })
+
+        i = i+1
+    end
+end

modules/dev/url-scan.lua

@@ -1,6 +1,7 @@
--- Description: Check subdomains for websites
+-- Description: Scan subdomains for websites
 -- Version: 0.1.0
 -- Source: subdomains
+-- License: GPL-3.0
 
 function request(subdomain_id, url)
     req = http_request(session, 'GET', url, {
@@ -13,12 +14,20 @@ function request(subdomain_id, url)
         return
     end
 
-    db_add('url', {
+    obj = {
         subdomain_id=subdomain_id,
         value=url,
         status=reply['status'],
         body=reply['text'],
-    })
+        redirect=reply['headers']['location'],
+    }
+
+    redirect = reply['headers']['location']
+    if redirect then
+        obj['redirect'] = url_join(url, redirect)
+    end
+
+    db_add('url', obj)
 
     -- info(json_encode(reply['status']))
     -- info(json_encode(reply['headers']['location']))

modules/dev/waybackurls.lua

@@ -1,6 +1,7 @@
 -- Description: Discover subdomains from wayback machine
 -- Version: 0.1.0
 -- Source: domains
+-- License: GPL-3.0
 
 function run(arg)
 

modules/harness/fail.lua

@@ -1,7 +1,9 @@
 -- Description: Test error handling
 -- Version: 0.1.0
 -- Source: domains
+-- License: GPL-3.0
 
 function run()
+    error('This is a non fatal error')
     return "This is an error: " .. 123
 end

modules/harness/sleep.lua

@@ -0,0 +1,8 @@
+-- Description: Sleep for 10 seconds
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run()
+    sleep(10)
+end

sn0int-registry/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "sn0int-registry"
-version = "0.1.0"
+version = "0.4.0"
 description = "sn0int registry"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 
 [dependencies]
-sn0int-common = { version="0.1.0", path="sn0int-common" }
+sn0int-common = { version="0.3.0", path="sn0int-common" }
 rocket = "0.3.16"
 rocket_codegen = "0.3.16"
 rocket_contrib = { version = "0.3.16", features = ["handlebars_templates"] }
@@ -22,8 +22,13 @@ failure = "0.1"
 url = "1.0"
 log = "0.4"
 semver = "0.9.0"
+lazy_static = "1.1"
+blake2 = "0.8.0"
+hex = "0.3.1"
+maplit = "1.0.1"
 
 serde = "1.0"
 serde_derive = "1.0"
+serde_json = "1.0"
 
 dotenv = "0.13"

sn0int-registry/assets/style.css

@@ -19,3 +19,17 @@ h1 a {
     text-decoration: none;
     color: #00aa00;
 }
+
+a {
+    color: #00aa00;
+}
+
+.code {
+    background-color: #222;
+    padding: 10px;
+}
+
+.list-unstyled {
+    list-style: none;
+    padding: 0 0 0 20px;
+}

sn0int-registry/sn0int-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.1.0"
+version = "0.3.0"
 description = "Common code for sn0int"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"

sn0int-registry/sn0int-common/src/metadata.rs

@@ -8,6 +8,7 @@ pub enum EntryType {
     Description,
     Version,
     Source,
+    License,
 }
 
 impl FromStr for EntryType {
@@ -18,6 +19,7 @@ impl FromStr for EntryType {
             "Description" => Ok(EntryType::Description),
             "Version" => Ok(EntryType::Version),
             "Source" => Ok(EntryType::Source),
+            "License" => Ok(EntryType::License),
             x => bail!("Unknown EntryType: {:?}", x),
         }
     }
@@ -27,6 +29,9 @@ impl FromStr for EntryType {
 pub enum Source {
     Domains,
     Subdomains,
+    IpAddrs,
+    Urls,
+    Emails,
 }
 
 impl FromStr for Source {
@@ -36,16 +41,46 @@ impl FromStr for Source {
         match s {
             "domains" => Ok(Source::Domains),
             "subdomains" => Ok(Source::Subdomains),
+            "ipaddrs" => Ok(Source::IpAddrs),
+            "urls" => Ok(Source::Urls),
+            "emails" => Ok(Source::Emails),
             x => bail!("Unknown Source: {:?}", x),
         }
     }
 }
 
+#[derive(Debug, PartialEq)]
+pub enum License {
+    MIT,
+    GPL3,
+    LGPL3,
+    BSD2,
+    BSD3,
+    WTFPL,
+}
+
+impl FromStr for License {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<License> {
+        match s {
+            "MIT" => Ok(License::MIT),
+            "GPL-3.0" => Ok(License::GPL3),
+            "LGPL-3.0" => Ok(License::LGPL3),
+            "BSD-2-Clause" => Ok(License::BSD2),
+            "BSD-3-Clause" => Ok(License::BSD3),
+            "WTFPL" => Ok(License::WTFPL),
+            x => bail!("Unsupported license: {:?}", x),
+        }
+    }
+}
+
 #[derive(Debug, PartialEq)]
 pub struct Metadata {
     pub description: String,
     pub version: String,
     pub source: Option<Source>,
+    pub license: License,
 }
 
 impl FromStr for Metadata {
@@ -62,6 +97,7 @@ impl FromStr for Metadata {
                 EntryType::Description => data.description = Some(v),
                 EntryType::Version => data.version = Some(v),
                 EntryType::Source => data.source = Some(v),
+                EntryType::License => data.license = Some(v),
             }
         }
 
@@ -74,6 +110,7 @@ pub struct NewMetadata<'a> {
     pub description: Option<&'a str>,
     pub version: Option<&'a str>,
     pub source: Option<&'a str>,
+    pub license: Option<&'a str>,
 }
 
 impl<'a> NewMetadata<'a> {
@@ -84,11 +121,14 @@ impl<'a> NewMetadata<'a> {
             Some(x) => Some(x.parse()?),
             _ => None,
         };
+        let license = self.license.ok_or_else(|| format_err!("License is required"))?;
+        let license = license.parse()?;
 
         Ok(Metadata {
             description: description.to_string(),
             version: version.to_string(),
             source,
+            license,
         })
     }
 }
@@ -99,9 +139,7 @@ named!(metaline<&str, (EntryType, &str)>, do_parse!(
     tag!(": ") >>
     value: take_until!("\n") >>
     tag!("\n") >>
-    (
-        (name, value)
-    )
+    (name, value)
 ));
 
 named!(metalines<&str, Vec<(EntryType, &str)>>, do_parse!(
@@ -122,12 +160,35 @@ mod tests {
         let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
 -- Version: 1.0.0
 -- Source: domains
+-- License: WTFPL
 
 "#).expect("parse");
         assert_eq!(metadata, Metadata {
             description: "Hello world, this is my description".to_string(),
             version: "1.0.0".to_string(),
+            license: License::WTFPL,
             source: Some(Source::Domains),
         });
     }
+
+    #[test]
+    fn verify_require_license() {
+        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+-- Version: 1.0.0
+-- Source: domains
+
+"#);
+        assert!(metadata.is_err());
+    }
+
+    #[test]
+    fn verify_require_opensource_license() {
+        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+-- Version: 1.0.0
+-- Source: domains
+-- License: Proprietary
+
+"#);
+        assert!(metadata.is_err());
+    }
 }

sn0int-registry/src/assets.rs

@@ -0,0 +1,13 @@
+use blake2::{Blake2b, Digest};
+
+
+pub static FAVICON: &[u8] = include_bytes!("../assets/favicon.ico");
+pub static STYLE_SHEET: &str = include_str!("../assets/style.css");
+
+lazy_static! {
+    pub static ref ASSET_REV: String = {
+        let mut h = Blake2b::new();
+        h.input(STYLE_SHEET.as_bytes());
+        hex::encode(&h.result()[0..8])
+    };
+}

sn0int-registry/src/db.rs

@@ -46,7 +46,7 @@ impl Deref for Connection {
     }
 }
 
-embed_migrations!("../sn0int-registry/migrations");
+embed_migrations!("migrations");
 
 pub fn wait_for_db(url: &str, attempts: u8) -> Result<PgConnection> {
     for _ in 0..attempts {

sn0int-registry/src/main.rs

@@ -8,9 +8,12 @@ extern crate sn0int_common;
 extern crate rocket;
 #[macro_use] extern crate rocket_contrib;
 extern crate dotenv;
-// extern crate serde;
+extern crate blake2;
+extern crate serde_json;
 #[macro_use] extern crate serde_derive;
 #[macro_use] extern crate log;
+#[macro_use] extern crate maplit;
+#[macro_use] extern crate lazy_static;
 #[macro_use] extern crate failure;
 #[macro_use] extern crate diesel;
 #[macro_use] extern crate diesel_migrations;
@@ -20,13 +23,13 @@ extern crate url;
 extern crate reqwest;
 extern crate semver;
 
-use rocket::response::content;
 use rocket_contrib::{Json, Value, Template};
 use dotenv::dotenv;
 
 use std::env;
 use errors::*;
 
+pub mod assets;
 pub mod auth;
 pub mod auth2;
 pub mod db;
@@ -38,21 +41,6 @@ pub mod routes;
 pub mod schema;
 
 
-#[get("/")]
-fn index() -> Template {
-    Template::render("index", ())
-}
-
-#[get("/favicon.ico")]
-fn favicon() -> Vec<u8> {
-    include_bytes!("../assets/favicon.ico").to_vec()
-}
-
-#[get("/assets/style.css")]
-fn style() -> content::Css<&'static str> {
-    content::Css(include_str!("../assets/style.css"))
-}
-
 #[catch(400)]
 fn bad_request() -> Json<Value> {
     Json(json!({
@@ -87,7 +75,7 @@ fn run() -> Result<()> {
         .manage(db::init(&database_url))
         .attach(Template::fairing())
         .mount("/api/v0", routes![
-            routes::api::dashboard,
+            routes::api::quickstart,
             routes::api::search,
             routes::api::info,
             routes::api::download,
@@ -100,9 +88,9 @@ fn run() -> Result<()> {
             routes::auth::login,
         ])
         .mount("/", routes![
-            index,
-            favicon,
-            style
+            routes::assets::index,
+            routes::assets::favicon,
+            routes::assets::style,
         ])
     .catch(catchers![
         bad_request,

sn0int-registry/src/models.rs

@@ -182,6 +182,18 @@ impl Module {
             downloads,
         )).collect())
     }
+
+    pub fn quickstart(connection: &PgConnection) -> Result<Vec<Module>> {
+        modules::table
+            .select(ALL_MODULE_COLUMNS)
+            .filter(modules::featured)
+            .order((
+                modules::author.asc(),
+                modules::name.asc(),
+            ))
+            .load(connection)
+            .map_err(Error::from)
+    }
 }
 
 #[derive(Insertable)]

sn0int-registry/src/routes/api.rs

@@ -6,13 +6,14 @@ use semver::Version;
 use sn0int_common::api::*;
 use sn0int_common::id;
 use sn0int_common::metadata::Metadata;
-use rocket_contrib::{Json, Value};
+use rocket_contrib::Json;
 use models::*;
 
 
-#[get("/dashboard")]
-fn dashboard() -> Json<Value> {
-    Json(json!({ "dashboard": {}}))
+#[get("/quickstart")]
+fn quickstart(connection: db::Connection) -> ApiResult<Json<ApiResponse<Vec<Module>>>> {
+    let modules = Module::quickstart(&connection)?;
+    Ok(Json(ApiResponse::Success(modules)))
 }
 
 #[derive(Debug, FromForm)]

sn0int-registry/src/routes/assets.rs

@@ -0,0 +1,26 @@
+use rocket::http::Status;
+use rocket::response::content;
+use rocket_contrib::Template;
+use assets::{ASSET_REV, FAVICON, STYLE_SHEET};
+
+
+#[get("/")]
+fn index() -> Template {
+    Template::render("index", hashmap!{
+        "ASSET_REV" => ASSET_REV.as_str(),
+    })
+}
+
+#[get("/favicon.ico")]
+fn favicon() -> Vec<u8> {
+    FAVICON.to_vec()
+}
+
+#[get("/assets/<rev>/style.css")]
+fn style(rev: String) -> Result<content::Css<&'static str>, Status> {
+    if rev == *ASSET_REV {
+        Ok(content::Css(STYLE_SHEET))
+    } else {
+        Err(Status::NotFound)
+    }
+}

sn0int-registry/src/routes/auth.rs

@@ -4,10 +4,16 @@ use db;
 use rocket::request::Form;
 use rocket::response::Redirect;
 use rocket_contrib::Template;
+use assets::ASSET_REV;
+use serde_json::{self, Value};
 
 
 #[get("/?<auth>")]
 pub fn get(auth: OAuth) -> Template {
+    let mut auth = serde_json::to_value(&auth).expect("OAuth serialization failed");
+    if let Value::Object(ref mut map) = auth {
+        map.insert("ASSET_REV".to_string(), Value::String(ASSET_REV.to_string()));
+    }
     Template::render("auth-confirm", auth)
 }
 
@@ -17,7 +23,9 @@ pub fn post(auth: Form<OAuth>, connection: db::Connection) -> ApiResult<Template
     let client = Authenticator::from_env()?;
     client.store_code(code, state, &connection)?;
 
-    Ok(Template::render("auth-done", vec![1]))
+    Ok(Template::render("auth-done", hashmap!{
+        "ASSET_REV" => ASSET_REV.as_str(),
+    }))
 }
 
 #[get("/<session>")]

sn0int-registry/src/routes/mod.rs

@@ -1,2 +1,3 @@
 pub mod api;
+pub mod assets;
 pub mod auth;

sn0int-registry/templates/base.html.hbs

@@ -3,7 +3,7 @@
     <head>
         <meta charset="utf-8">
         <title>sn0int registry</title>
-        <link rel="stylesheet" href="/assets/style.css">
+        <link rel="stylesheet" href="/assets/{{ASSET_REV}}/style.css">
     </head>
     <body>
         <h1><a href="/">sn0int</a></h1>

sn0int-registry/templates/index.html.hbs

@@ -1,6 +1,74 @@
 {{#*inline "page"}}
         <p>
-        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+This is the registry server of sn0int, an OSINT framework and package manager.
+It was built for IT security professionals and bug hunters to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations.
+        </p>
+
+        <p>
+Among other things, sn0int is currently able to:
+        </p>
+
+        <ul class="list-unstyled">
+<li>[X] Harvest subdomains from certificate transparency logs</li>
+<li>[X] Harvest subdomains from various passive dns logs</li>
+<li>[X] Sift through subdomain results for publicly accessible websites</li>
+<li>[X] Harvest emails from pgp keyservers</li>
+<li>[X] Enrich ip addresses with ASN and geoip info</li>
+<li>[X] Harvest subdomains from the wayback machine</li>
+        </ul>
+
+        <p>
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource.  None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
+        </p>
+
+        <p>
+        The registry server allows uploading modules that can be installed by
+        users afterwards. This is done using sn0int which needs to be
+        <a href="https://github.com/kpcyrd/sn0int#installation" target="_blank"
+        rel="noopener">installed</a>.
+        </p>
+
+        <p>
+        You can search for modules with sn0int using:
+        </p>
+        <p class="code"><code>
+        sn0int search ctlogs
+        </code></p>
+
+        <p>
+        To install the module afterwards run:
+        </p>
+        <p class="code"><code>
+        sn0int install kpcyrd/ctlogs
+        </code></p>
+
+        <p>
+        If you want to publish your own module, you need to login with github first:
+        </p>
+        <p class="code"><code>
+        sn0int login
+        </code></p>
+
+        <p>
+        Afterwards you can upload your module to the registry (this is going to
+        publish your module as yourgithubuser/example):
+        </p>
+        <p class="code"><code>
+        sn0int publish ./path/to/example.lua
+        </code></p>
+
+        <p>
+        For more information, visit <a href="https://github.com/kpcyrd/sn0int"
+        target="_blank" rel="noopener">sn0int on github</a>.
         </p>
 {{/inline}}
 {{~> base }}

src/api.rs

@@ -1,5 +1,6 @@
 use errors::*;
 use std::fmt;
+use config::Config;
 use chrootable_https::{self, HttpClient, Body, Request, Uri};
 use chrootable_https::http::request::Builder as RequestBuilder;
 use chrootable_https::header::CONTENT_TYPE;
@@ -12,8 +13,6 @@ use sn0int_common::api::*;
 use sn0int_common::ModuleID;
 use web;
 
-pub const API_URL: &str = "http://[::1]:8000";
-
 
 pub struct Client {
     server: String,
@@ -22,10 +21,10 @@ pub struct Client {
 }
 
 impl Client {
-    pub fn new<I: Into<String>>(server: I) -> Result<Client> {
+    pub fn new(config: &Config) -> Result<Client> {
         let client = chrootable_https::Client::with_system_resolver()?;
         Ok(Client {
-            server: server.into(),
+            server: config.core.registry.clone(),
             client,
             session: None,
         })
@@ -98,7 +97,7 @@ impl Client {
     pub fn publish_module(&self, name: &str, body: String) -> Result<PublishResponse> {
         let url = format!("{}/api/v0/publish/{}", self.server, name);
         let reply = self.post::<PublishResponse, _>(&url, &PublishRequest {
-            code: body.into(),
+            code: body,
         })?;
         Ok(reply)
     }
@@ -122,4 +121,10 @@ impl Client {
         })?;
         Ok(reply)
     }
+
+    pub fn quickstart(&self) -> Result<Vec<ModuleInfoResponse>> {
+        let url = format!("{}/api/v0/quickstart", self.server);
+        let reply = self.get::<Vec<ModuleInfoResponse>>(&url)?;
+        Ok(reply)
+    }
 }

src/archive.rs

@@ -0,0 +1,40 @@
+use errors::*;
+use tar;
+use libflate::gzip;
+use std::io;
+use std::fs::File;
+use std::path::Path;
+
+
+pub fn extract<R: io::Read, P: AsRef<Path>>(read: &mut R, filter: &str, target: P) -> Result<()> {
+    let file = gzip::Decoder::new(read)?;
+    let mut ar = tar::Archive::new(file);
+
+    for entry in ar.entries()? {
+        let mut entry = entry?;
+        let file_name = {
+            let path = entry.path()?;
+            path.file_name()
+                .ok_or_else(|| format_err!("Invalid path in archive"))?
+                .to_str()
+                .ok_or_else(|| format_err!("Filename is invalid utf8"))?
+                .to_owned()
+        };
+
+        debug!("Found in archive: {:?}", file_name);
+
+        if filter == file_name {
+            debug!("Extracting to {:?}", target.as_ref());
+
+            let mut target = File::create(target)?;
+            let n = io::copy(&mut entry, &mut target)?;
+            debug!("Wrote {:?} bytes", n);
+
+            return Ok(());
+        }
+
+        debug!("Skipping file");
+    }
+
+    bail!("Nothing in archive matched filter")
+}

src/args.rs

@@ -1,5 +1,6 @@
-use structopt::clap::AppSettings;
+use structopt::clap::{AppSettings, Shell};
 use sn0int_common::ModuleID;
+use workspaces::Workspace;
 
 
 #[derive(Debug, StructOpt)]
@@ -8,7 +9,7 @@ use sn0int_common::ModuleID;
 pub struct Args {
     #[structopt(short="w", long="workspace")]
     /// Select a different workspace instead of the default
-    pub workspace: Option<String>,
+    pub workspace: Option<Workspace>,
 
     #[structopt(subcommand)]
     pub subcommand: Option<SubCommand>,
@@ -43,6 +44,9 @@ pub enum SubCommand {
     #[structopt(author="", name="search")]
     /// Search in the registry
     Search(Search),
+    #[structopt(author="", name="completions")]
+    /// Generate shell completions
+    Completions(Completions),
 }
 
 #[derive(Debug, StructOpt)]
@@ -50,6 +54,8 @@ pub struct Run {
     pub module: Option<String>,
     #[structopt(short="f", long="file", conflicts_with="module")]
     pub file: Option<String>,
+    #[structopt(short="j", long="threads", default_value="1")]
+    pub threads: usize,
 }
 
 #[derive(Debug, StructOpt)]
@@ -64,8 +70,9 @@ pub struct Login {
 
 #[derive(Debug, StructOpt)]
 pub struct Publish {
-    /// The script to publish
-    pub path: String,
+    /// The scripts to publish
+    #[structopt(raw(required = "true"))]
+    pub paths: Vec<String>,
 }
 
 #[derive(Debug, StructOpt)]
@@ -81,3 +88,9 @@ pub struct Search {
     /// The search query
     pub query: String,
 }
+
+#[derive(Debug, StructOpt)]
+pub struct Completions {
+    #[structopt(raw(possible_values="&Shell::variants()"))]
+    pub shell: Shell,
+}

src/auth.rs

@@ -3,7 +3,8 @@ use opener;
 use std::fs;
 use std::thread;
 use std::time::Duration;
-use api::{API_URL, Client};
+use api::Client;
+use config::Config;
 use paths;
 use term;
 
@@ -22,8 +23,8 @@ pub fn save_token(session: &str) -> Result<()> {
     Ok(())
 }
 
-pub fn run_login() -> Result<()> {
-    let mut client = Client::new(API_URL)?;
+pub fn run_login(config: &Config) -> Result<()> {
+    let mut client = Client::new(config)?;
 
     if let Ok(session) = load_token() {
         client.authenticate(session);
@@ -35,7 +36,7 @@ pub fn run_login() -> Result<()> {
 
     let session = Client::random_session();
     client.authenticate(session.clone());
-    let url = format!("{}/auth/{}", API_URL, session);
+    let url = format!("{}/auth/{}", config.core.registry, session);
 
     term::success(&format!("Opening url: {}", url));
     opener::open(url)?;

src/cmd/add_cmd.rs

@@ -1,5 +1,6 @@
 use errors::*;
 
+use models::*;
 use shell::Readline;
 use structopt::StructOpt;
 use utils;
@@ -11,6 +12,8 @@ pub enum Args {
     Domain(AddDomain),
     #[structopt(name="subdomain")]
     Subdomain(AddSubdomain),
+    #[structopt(name="email")]
+    Email(AddEmail),
 }
 
 #[derive(Debug, StructOpt)]
@@ -23,11 +26,17 @@ pub struct AddSubdomain {
     subdomain: Option<String>,
 }
 
+#[derive(Debug, StructOpt)]
+pub struct AddEmail {
+    email: Option<String>,
+}
+
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
     match args {
         Args::Domain(args) => add_domain(rl, args),
         Args::Subdomain(args) => add_subdomain(rl, args),
+        Args::Email(args) => add_email(rl, args),
     }
 }
 
@@ -45,7 +54,9 @@ fn add_domain(rl: &mut Readline, args: AddDomain) -> Result<()> {
         bail!("This is not a valid domain, might be a subdomain or tld");
     }
 
-    rl.db().insert_domain(&domain)?;
+    rl.db().insert_struct(NewDomain {
+        value: &domain,
+    })?;
 
     Ok(())
 }
@@ -63,7 +74,32 @@ fn add_subdomain(rl: &mut Readline, args: AddSubdomain) -> Result<()> {
         .ok_or_else(|| format_err!("Dns Name seems invalid"))?
         .to_string();
 
-    rl.db().insert_subdomain(&subdomain, &domain)?;
+    let domain_id = match rl.db().insert_struct(NewDomain {
+        value: &domain,
+    })? {
+        Some((_, domain_id)) => domain_id,
+        _ => bail!("Domain is out out of scope"),
+    };
+
+    rl.db().insert_struct(NewSubdomain {
+        domain_id,
+        value: &subdomain,
+        resolvable: None,
+    })?;
+
+    Ok(())
+}
+
+fn add_email(rl: &mut Readline, args: AddEmail) -> Result<()> {
+    let email = match args.email {
+        Some(email) => email,
+        _ => utils::question("Email")?,
+    };
+
+    rl.db().insert_struct(NewEmail {
+        value: &email,
+        valid: None,
+    })?;
 
     Ok(())
 }

src/cmd/delete_cmd.rs

@@ -0,0 +1,50 @@
+use errors::*;
+
+use db;
+use shell::Readline;
+use structopt::StructOpt;
+use models::*;
+use term;
+
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="domains")]
+    Domains(Filter),
+    #[structopt(name="subdomains")]
+    Subdomains(Filter),
+    #[structopt(name="ipaddrs")]
+    IpAddrs(Filter),
+    #[structopt(name="urls")]
+    Urls(Filter),
+    #[structopt(name="emails")]
+    Emails(Filter),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Filter {
+    args: Vec<String>,
+}
+
+impl Filter {
+    pub fn parse(&self) -> Result<db::Filter> {
+        db::Filter::parse(&self.args)
+    }
+}
+
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    let args = Args::from_iter_safe(args)?;
+    let rows = match args {
+        Args::Domains(filter) => delete::<Domain>(rl, &filter),
+        Args::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
+        Args::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
+        Args::Urls(filter) => delete::<Url>(rl, &filter),
+        Args::Emails(filter) => delete::<Email>(rl, &filter),
+    }?;
+    term::info(&format!("Deleted {} rows", rows));
+    Ok(())
+}
+
+fn delete<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<usize> {
+    rl.db().delete::<T>(&filter.parse()?)
+}

src/cmd/mod.rs

@@ -1,6 +1,11 @@
 pub mod add_cmd;
-pub mod switch_db_cmd;
+pub mod delete_cmd;
 pub mod run_cmd;
 pub mod use_cmd;
 pub mod select_cmd;
 pub mod mod_cmd;
+pub mod noscope_cmd;
+pub mod scope_cmd;
+pub mod target_cmd;
+pub mod quickstart_cmd;
+pub mod workspace_cmd;

src/cmd/mod_cmd.rs

@@ -1,10 +1,16 @@
 use errors::*;
 
 use args;
+use args::Install;
+use api::Client;
+use config::Config;
 use colored::Colorize;
+use engine::Module;
 use registry;
 use shell::Readline;
 use structopt::StructOpt;
+use term;
+use worker;
 
 
 #[derive(Debug, StructOpt)]
@@ -27,6 +33,9 @@ pub enum SubCommand {
     #[structopt(author="", name="reload")]
     /// Reload modules
     Reload(Reload),
+    #[structopt(author="", name="update")]
+    /// Update modules
+    Update(Update),
 }
 
 #[derive(Debug, StructOpt)]
@@ -37,8 +46,31 @@ pub struct List {
 pub struct Reload {
 }
 
+#[derive(Debug, StructOpt)]
+pub struct Update {
+}
+
+fn update(client: &Client, config: &Config, module: &Module) -> Result<()> {
+    let installed = module.version();
+    let infos = client.query_module(&module.id())?;
+    debug!("Latest version: {:?}", infos);
+
+    let latest = infos.latest.ok_or_else(|| format_err!("Module doesn't have any released versions"))?;
+
+    if installed != latest {
+        term::info(&format!("Updating {}: {:?} -> {:?}", module.canonical(), installed, latest));
+        registry::run_install(&Install {
+            module: module.id(),
+            version: None,
+        }, &config)?;
+    }
+
+    Ok(())
+}
+
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
+    let config = rl.config().clone();
 
     match args.subcommand {
         SubCommand::List(_) => {
@@ -49,11 +81,11 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
             }
         },
         SubCommand::Install(install) => {
-            registry::run_install(&install)?;
+            registry::run_install(&install, &config)?;
             // trigger reload
             run(rl, &[String::from("mod"), String::from("reload")])?;
         },
-        SubCommand::Search(search) => registry::run_search(&search)?,
+        SubCommand::Search(search) => registry::run_search(&search, &config)?,
         SubCommand::Reload(_) => {
             let current = rl.take_module()
                             .map(|m| m.canonical());
@@ -67,6 +99,25 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
                 }
             }
         },
+        SubCommand::Update(_) => {
+            let client = Client::new(&config)?;
+
+            for module in rl.engine().list() {
+                let name = module.canonical();
+                let label = format!("Searching for updates {}", name);
+
+                let result = worker::spawn_fn(&label, || {
+                    update(&client, &config, &module)
+                }, true);
+
+                if let Err(err) = result {
+                    term::error(&format!("Failed to update {}: {:?}", name, err));
+                }
+            }
+
+            // trigger reload
+            run(rl, &[String::from("mod"), String::from("reload")])?;
+        },
     }
 
     Ok(())

src/cmd/noscope_cmd.rs

@@ -0,0 +1,50 @@
+use errors::*;
+
+use db;
+use shell::Readline;
+use structopt::StructOpt;
+use models::*;
+use term;
+
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="domains")]
+    Domains(Filter),
+    #[structopt(name="subdomains")]
+    Subdomains(Filter),
+    #[structopt(name="ipaddrs")]
+    IpAddrs(Filter),
+    #[structopt(name="urls")]
+    Urls(Filter),
+    #[structopt(name="emails")]
+    Emails(Filter),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Filter {
+    args: Vec<String>,
+}
+
+impl Filter {
+    pub fn parse(&self) -> Result<db::Filter> {
+        db::Filter::parse(&self.args)
+    }
+}
+
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    let args = Args::from_iter_safe(args)?;
+    let rows = match args {
+        Args::Domains(filter) => noscope::<Domain>(rl, &filter),
+        Args::Subdomains(filter) => noscope::<Subdomain>(rl, &filter),
+        Args::IpAddrs(filter) => noscope::<IpAddr>(rl, &filter),
+        Args::Urls(filter) => noscope::<Url>(rl, &filter),
+        Args::Emails(filter) => noscope::<Email>(rl, &filter),
+    }?;
+    term::info(&format!("Updated {} rows", rows));
+    Ok(())
+}
+
+fn noscope<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<usize> {
+    rl.db().noscope::<T>(&filter.parse()?)
+}

src/cmd/quickstart_cmd.rs

@@ -0,0 +1,38 @@
+use errors::*;
+
+use args::Install;
+use api::Client;
+// use colored::Colorize;
+use cmd::mod_cmd;
+use registry;
+use shell::Readline;
+use structopt::StructOpt;
+use sn0int_common::ModuleID;
+
+
+#[derive(Debug, StructOpt)]
+pub struct Args {
+}
+
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    let _args = Args::from_iter_safe(args)?;
+    let config = rl.config().clone();
+
+    let client = Client::new(&config)?;
+
+    for module in client.quickstart()? {
+        info!("Installing {:?}", module);
+        registry::run_install(&Install {
+            module: ModuleID {
+                author: module.author,
+                name: module.name,
+            },
+            version: None,
+        }, &config)?;
+    }
+
+    // trigger reload
+    mod_cmd::run(rl, &[String::from("mod"), String::from("reload")])?;
+
+    Ok(())
+}

src/cmd/run_cmd.rs

@@ -1,11 +1,10 @@
 use errors::*;
 
-use db::Database;
+use db::{Database, Filter};
 use sn0int_common::metadata::Source;
 use serde::Serialize;
 use serde_json;
 use shell::Readline;
-use std::fmt;
 use structopt::StructOpt;
 use models::*;
 use term;
@@ -14,46 +13,49 @@ use worker;
 
 #[derive(Debug, StructOpt)]
 pub struct Args {
+    #[structopt(short="j", long="threads", default_value="1")]
+    threads: usize,
 }
 
-fn prepare_arg<T: Serialize + fmt::Display>(x: T) -> Result<(serde_json::Value, Option<String>)> {
+fn prepare_arg<T: Serialize + Model>(x: T) -> Result<(serde_json::Value, Option<String>)> {
     let pretty = x.to_string();
     let arg = serde_json::to_value(x)?;
     Ok((arg, Some(pretty)))
 }
 
-fn prepare_args<T: Model + Serialize + fmt::Display>(db: &Database) -> Result<Vec<(serde_json::Value, Option<String>)>> {
-    db.list::<T>()?
+fn prepare_args<T: Scopable + Serialize + Model>(db: &Database, filter: &Filter) -> Result<Vec<(serde_json::Value, Option<String>)>> {
+    db.filter::<T>(filter)?
         .into_iter()
         .map(prepare_arg)
         .collect()
 }
 
-pub fn execute(rl: &mut Readline) -> Result<()> {
+pub fn execute(rl: &mut Readline, threads: usize) -> Result<()> {
     let module = rl.module()
         .map(|m| m.to_owned())
         .ok_or_else(|| format_err!("No module selected"))?;
 
-    let args = match module.source() {
-        Some(Source::Domains) => prepare_args::<Domain>(rl.db()),
-        Some(Source::Subdomains) => prepare_args::<Subdomain>(rl.db()),
-        None => Ok(vec![(serde_json::Value::Null, None)]),
-    };
+    let filter = rl.scoped_targets();
+
+    let args = match module.source() {
+        Some(Source::Domains) => prepare_args::<Domain>(rl.db(), &filter),
+        Some(Source::Subdomains) => prepare_args::<Subdomain>(rl.db(), &filter),
+        Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl.db(), &filter),
+        Some(Source::Urls) => prepare_args::<Url>(rl.db(), &filter),
+        Some(Source::Emails) => prepare_args::<Email>(rl.db(), &filter),
+        None => Ok(vec![(serde_json::Value::Null, None)]),
+    }?;
+
+    rl.signal_register().catch_ctrl();
+    worker::spawn(rl, &module, args, threads);
+    rl.signal_register().reset_ctrlc();
 
-    rl.catch_ctrl();
-    for (arg, pretty_arg) in args? {
-        worker::spawn(rl, module.clone(), arg, &pretty_arg);
-        if rl.ctrlc_received() {
-            break;
-        }
-    }
-    rl.reset_ctrlc();
     term::info(&format!("Finished {}", module.canonical()));
 
     Ok(())
 }
 
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
-    let _args = Args::from_iter_safe(args)?;
-    execute(rl)
+    let args = Args::from_iter_safe(args)?;
+    execute(rl, args.threads)
 }

src/cmd/scope_cmd.rs

@@ -0,0 +1,50 @@
+use errors::*;
+
+use db;
+use shell::Readline;
+use structopt::StructOpt;
+use models::*;
+use term;
+
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="domains")]
+    Domains(Filter),
+    #[structopt(name="subdomains")]
+    Subdomains(Filter),
+    #[structopt(name="ipaddrs")]
+    IpAddrs(Filter),
+    #[structopt(name="urls")]
+    Urls(Filter),
+    #[structopt(name="emails")]
+    Emails(Filter),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Filter {
+    args: Vec<String>,
+}
+
+impl Filter {
+    pub fn parse(&self) -> Result<db::Filter> {
+        db::Filter::parse(&self.args)
+    }
+}
+
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    let args = Args::from_iter_safe(args)?;
+    let rows = match args {
+        Args::Domains(filter) => scope::<Domain>(rl, &filter),
+        Args::Subdomains(filter) => scope::<Subdomain>(rl, &filter),
+        Args::IpAddrs(filter) => scope::<IpAddr>(rl, &filter),
+        Args::Urls(filter) => scope::<Url>(rl, &filter),
+        Args::Emails(filter) => scope::<Email>(rl, &filter),
+    }?;
+    term::info(&format!("Updated {} rows", rows));
+    Ok(())
+}
+
+fn scope<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<usize> {
+    rl.db().scope::<T>(&filter.parse()?)
+}

src/cmd/select_cmd.rs

@@ -16,6 +16,8 @@ pub enum Args {
     IpAddrs(Filter),
     #[structopt(name="urls")]
     Urls(Filter),
+    #[structopt(name="emails")]
+    Emails(Filter),
 }
 
 #[derive(Debug, StructOpt)]
@@ -25,7 +27,7 @@ pub struct Filter {
 
 impl Filter {
     pub fn parse(&self) -> Result<db::Filter> {
-        db::Filter::parse(&self.args)
+        db::Filter::parse_optional(&self.args)
     }
 }
 
@@ -36,6 +38,7 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
         Args::Subdomains(filter) => select::<Subdomain>(rl, &filter),
         Args::IpAddrs(filter) => select::<IpAddr>(rl, &filter),
         Args::Urls(filter) => select::<Url>(rl, &filter),
+        Args::Emails(filter) => select::<Email>(rl, &filter),
     }
 }
 

src/cmd/switch_db_cmd.rs

@@ -1,18 +0,0 @@
-use errors::*;
-
-use db::Database;
-use shell::Readline;
-use structopt::StructOpt;
-
-
-#[derive(Debug, StructOpt)]
-pub struct Args {
-    workspace: String,
-}
-
-pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    let db = Database::establish(args.workspace)?;
-    rl.set_db(db);
-    Ok(())
-}

src/cmd/target_cmd.rs

@@ -0,0 +1,69 @@
+use errors::*;
+
+use db;
+use shell::Readline;
+use sn0int_common::metadata::Source;
+use structopt::StructOpt;
+use term;
+use models::*;
+
+
+#[derive(Debug, StructOpt)]
+pub struct Args {
+    // TODO: target -p # print current filter
+    // TODO: target -c # clear filter
+
+    filter: Vec<String>,
+}
+
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    let args = Args::from_iter_safe(args)?;
+
+    let source = rl.module()
+        .ok_or_else(|| format_err!("No module selected"))
+        .map(|x| x.source().clone())?;
+
+    let source = source
+        .ok_or_else(|| format_err!("Module doesn't have sources"))?;
+
+    if args.filter.is_empty() {
+        match source {
+            Source::Domains => select::<Domain>(rl)?,
+            Source::Subdomains => select::<Subdomain>(rl)?,
+            Source::IpAddrs => select::<IpAddr>(rl)?,
+            Source::Urls => select::<Url>(rl)?,
+            Source::Emails => select::<Email>(rl)?,
+        }
+    } else {
+        debug!("Setting filter to {:?}", args.filter);
+        let filter = db::Filter::parse_optional(&args.filter)?;
+        rl.set_target(Some(filter));
+        term::info(&format!("{} entities selected", count_selected(rl, &source)?));
+    }
+
+    Ok(())
+}
+
+fn count_selected(rl: &mut Readline, source: &Source) -> Result<usize> {
+    let db = rl.db();
+    let filter = rl.scoped_targets();
+
+    let num = match source {
+        Source::Domains => db.filter::<Domain>(&filter)?.len(),
+        Source::Subdomains => db.filter::<Subdomain>(&filter)?.len(),
+        Source::IpAddrs => db.filter::<IpAddr>(&filter)?.len(),
+        Source::Urls => db.filter::<Url>(&filter)?.len(),
+        Source::Emails => db.filter::<Email>(&filter)?.len(),
+    };
+    Ok(num)
+}
+
+fn select<T: Model + Detailed>(rl: &mut Readline) -> Result<()> {
+    let filter = rl.scoped_targets();
+
+    for obj in rl.db().filter::<T>(&filter)? {
+        println!("{}", obj.detailed(rl.db())?);
+    }
+
+    Ok(())
+}

src/cmd/workspace_cmd.rs

@@ -0,0 +1,27 @@
+use errors::*;
+
+use db::Database;
+use shell::Readline;
+use structopt::StructOpt;
+use workspaces::{self, Workspace};
+
+
+#[derive(Debug, StructOpt)]
+pub struct Args {
+    workspace: Option<Workspace>,
+}
+
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    let args = Args::from_iter_safe(args)?;
+
+    if let Some(workspace) = args.workspace {
+        let db = Database::establish(workspace)?;
+        rl.set_db(db);
+    } else {
+        for x in workspaces::list()? {
+            println!("{:?}", x);
+        }
+    }
+
+    Ok(())
+}

src/complete.rs

@@ -1,3 +1,5 @@
+use args::{Args, Completions};
+use errors::*;
 use rustyline;
 use rustyline::completion::Completer;
 use rustyline::highlight::Highlighter;
@@ -5,7 +7,10 @@ use rustyline::hint::Hinter;
 use shellwords;
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::str::FromStr;
+use std::io::stdout;
+use structopt::StructOpt;
 use shell::Command;
+use workspaces;
 
 
 #[derive(Debug, Default)]
@@ -14,6 +19,26 @@ pub struct CmdCompleter {
 }
 
 impl CmdCompleter {
+    pub fn filter(&self, cmd: &str, args: &[String]) -> rustyline::Result<(usize, Vec<String>)> {
+        // we can only complete the 2nd argument
+        if args.len() != 2 {
+            Ok((0, vec![]))
+        } else {
+            let arg = &args[1];
+
+            let options = &["domains",
+                            "subdomains",
+                            "ipaddrs",
+                            "urls",
+                            "emails"];
+
+            let results: Vec<String> = options.iter()
+                .filter(|x| x.starts_with(arg))
+                .map(|x| format!("{} {} ", cmd, x))
+                .collect();
+            Ok((0, results))
+        }
+    }
 }
 
 impl Completer for CmdCompleter {
@@ -70,6 +95,7 @@ impl Completer for CmdCompleter {
                         Ok((0, results))
                     }
                 },
+                Command::Delete => self.filter("delete", &cmd),
                 Command::Mod => {
                     // we can only complete the 2nd argument
                     if args != 2 {
@@ -90,6 +116,7 @@ impl Completer for CmdCompleter {
                     }
 
                 },
+                Command::Noscope => self.filter("noscope", &cmd),
                 Command::Use => {
                     // we can only complete the 2nd argument
                     if args != 2 {
@@ -104,22 +131,23 @@ impl Completer for CmdCompleter {
                         Ok((0, results))
                     }
                 },
-                Command::Select => {
+                Command::Scope => self.filter("scope", &cmd),
+                Command::Select => self.filter("select", &cmd),
+                Command::Workspace => {
                     // we can only complete the 2nd argument
                     if args != 2 {
                         Ok((0, vec![]))
                     } else {
                         let arg = &cmd[1];
 
-                        let options = &["domains",
-                                        "subdomains",
-                                        "ipaddrs",
-                                        "urls"];
+                        let results: Vec<String> = match workspaces::list() {
+                            Ok(workspaces) => workspaces.iter()
+                                .filter(|x| x.starts_with(arg))
+                                .map(|x| format!("workspace {} ", x.as_str()))
+                                .collect(),
+                            _ => Vec::new(),
+                        };
 
-                        let results: Vec<String> = options.iter()
-                            .filter(|x| x.starts_with(arg))
-                            .map(|x| format!("select {} ", x))
-                            .collect();
                         Ok((0, results))
                     }
                 },
@@ -160,3 +188,8 @@ impl Highlighter for CmdCompleter {
 }
 
 impl rustyline::Helper for CmdCompleter {}
+
+pub fn run_generate(args: &Completions) -> Result<()> {
+    Args::clap().gen_completions_to("sn0int", args.shell, &mut stdout());
+    Ok(())
+}

src/config.rs

@@ -0,0 +1,62 @@
+use dirs;
+use errors::*;
+use std::fs;
+use std::path::{Path, PathBuf};
+use toml;
+
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct Config {
+    pub core: CoreConfig,
+}
+
+impl Config {
+    pub fn load() -> Result<Config> {
+        let path = Config::path()?;
+        Config::load_from(&path)
+    }
+
+    pub fn path() -> Result<PathBuf> {
+        let path = dirs::config_dir()
+            .ok_or_else(|| format_err!("Failed to find config directory"))?;
+        let path = path.join("sn0int.toml");
+        Ok(path)
+    }
+
+    pub fn load_or_default() -> Result<Config> {
+        let path = Config::path()?;
+        if path.exists() {
+            Config::load_from(&path)
+        } else {
+            Ok(Config::default())
+        }
+    }
+
+    pub fn load_from<P: AsRef<Path>>(path: P) -> Result<Config> {
+        let config = fs::read(&path)
+            .context("Failed to read config file")?;
+
+        let config = toml::from_slice(&config)?;
+
+        Ok(config)
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CoreConfig {
+    #[serde(default="default_registry")]
+    pub registry: String,
+}
+
+impl Default for CoreConfig {
+    fn default() -> CoreConfig {
+        CoreConfig {
+            registry: default_registry(),
+        }
+    }
+}
+
+#[inline]
+fn default_registry() -> String {
+    String::from("https://sn0int.com")
+}

src/crt.rs

@@ -0,0 +1,297 @@
+use errors::*;
+
+use x509_parser;
+use der_parser::oid::Oid;
+use std::collections::HashSet;
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
+use nom::be_u8;
+
+
+#[derive(Debug, PartialEq)]
+pub enum AlternativeName {
+    DnsName(String),
+    IpAddr(IpAddr),
+}
+
+named!(san_extension<&[u8], Vec<AlternativeName>>, do_parse!(
+    _tag:   tag!(b"\x30")   >>
+    len:    be_u8           >>
+    values: take!(len)      >>
+    ({
+        let mut bytes = values;
+        let mut values = Vec::new();
+        while !bytes.is_empty() {
+            let (rem, v) = san_value(bytes)?;
+            match v {
+                Ok(v) => values.push(v),
+                Err(e) => warn!("Unknown field in SAN extension: {}", e),
+            }
+            bytes = rem;
+        }
+        values
+    })
+));
+
+named!(san_value<&[u8], Result<AlternativeName>>,
+    switch!(be_u8,
+        0x82    => call!(san_value_dns) |
+        0x87    => call!(san_value_ipaddr) |
+        k       => call!(san_value_unknown, k)
+    )
+);
+
+named!(san_value_dns<&[u8], Result<AlternativeName>>, do_parse!(
+    len:    be_u8           >>
+    value:  take!(len)      >>
+    ({
+        String::from_utf8(value.to_vec())
+            .map(|v| AlternativeName::DnsName(v))
+            .map_err(Error::from)
+    })
+));
+
+named!(san_value_ipaddr<&[u8], Result<AlternativeName>>, do_parse!(
+    len:    be_u8           >>
+    v:      take!(len)      >>
+    ({
+        match len {
+            4  => Ok(AlternativeName::IpAddr(Ipv4Addr::from([
+                v[0], v[1], v[2], v[3],
+            ]).into())),
+            16 => Ok(AlternativeName::IpAddr(Ipv6Addr::from([
+                v[0],  v[1],  v[2],  v[3],
+                v[4],  v[5],  v[6],  v[7],
+                v[8],  v[9],  v[10], v[11],
+                v[12], v[13], v[14], v[15],
+            ]).into())),
+            _ => Err(format_err!("Invalid ipaddr")),
+        }
+    })
+));
+
+named_args!(san_value_unknown(key: u8)<&[u8], Result<AlternativeName>>, do_parse!(
+    len:    be_u8           >>
+    v:      take!(len)      >>
+    ({
+        Err(format_err!("Unexpected type {:?} => {:?}", key, v))
+    })
+));
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct Certificate {
+    pub valid_names: Vec<String>,
+    pub valid_ipaddrs: Vec<IpAddr>
+}
+
+impl Certificate {
+    pub fn parse_pem(crt: &str) -> Result<Certificate> {
+        let pem = match x509_parser::pem::pem_to_der(crt.as_bytes()) {
+            Ok((remaining, pem)) => {
+                if !remaining.is_empty() {
+                    bail!("input cert has trailing garbage");
+                }
+                if pem.label != "CERTIFICATE" {
+                    bail!("input is not a certificate");
+                }
+                pem
+            },
+            Err(_) => bail!("Failed to parse pem"),
+        };
+        Certificate::from_bytes(&pem.contents)
+    }
+
+    pub fn from_bytes(crt: &[u8]) -> Result<Certificate> {
+        let crt = match x509_parser::parse_x509_der(&crt) {
+            Ok((remaining, der)) => {
+                if !remaining.is_empty() {
+                    bail!("input cert has trailing garbage");
+                }
+                if der.tbs_certificate.version != 2 {
+                    bail!("unexpected certificate version");
+                }
+                der
+            },
+            Err(_) => bail!("Failed to parse der"),
+        };
+
+        let mut valid_names = HashSet::new();
+        let mut valid_ipaddrs = HashSet::new();
+
+        for x in crt.tbs_certificate.subject.rdn_seq {
+            for y in x.set {
+                // CommonName
+                if y.attr_type != Oid::from(&[2, 5, 4, 3]) {
+                    continue;
+                }
+
+                if let Ok(x) = y.attr_value.content.as_slice() {
+                    let value = String::from_utf8(x.to_vec())?;
+                    info!("Found CN in Subject: {:?}", value);
+                    valid_names.insert(value);
+                }
+            }
+        }
+
+        for x in crt.tbs_certificate.extensions {
+            if x.oid != Oid::from(&[2, 5, 29, 17]) {
+                continue;
+            }
+
+            debug!("Found san extension: {:?}", x.value);
+            let values = match san_extension(x.value) {
+                Ok((remaining, values)) => {
+                    if !remaining.is_empty() {
+                        bail!("san extension has trailing garbage");
+                    }
+                    values
+                },
+                Err(_) => bail!("Failed to parse san extension"),
+            };
+
+            for v in values {
+                match v {
+                    AlternativeName::DnsName(v) => valid_names.insert(v),
+                    AlternativeName::IpAddr(v) => valid_ipaddrs.insert(v),
+                };
+            }
+        }
+
+        let valid_names = valid_names.into_iter().collect();
+        let valid_ipaddrs = valid_ipaddrs.into_iter().collect();
+        Ok(Certificate {
+            valid_names,
+            valid_ipaddrs,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_pem_github() {
+        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+MIIHQjCCBiqgAwIBAgIQCgYwQn9bvO1pVzllk7ZFHzANBgkqhkiG9w0BAQsFADB1
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMTQwMgYDVQQDEytEaWdpQ2VydCBTSEEyIEV4dGVuZGVk
+IFZhbGlkYXRpb24gU2VydmVyIENBMB4XDTE4MDUwODAwMDAwMFoXDTIwMDYwMzEy
+MDAwMFowgccxHTAbBgNVBA8MFFByaXZhdGUgT3JnYW5pemF0aW9uMRMwEQYLKwYB
+BAGCNzwCAQMTAlVTMRkwFwYLKwYBBAGCNzwCAQITCERlbGF3YXJlMRAwDgYDVQQF
+Ewc1MTU3NTUwMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQG
+A1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMuMRMwEQYD
+VQQDEwpnaXRodWIuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+xjyq8jyXDDrBTyitcnB90865tWBzpHSbindG/XqYQkzFMBlXmqkzC+FdTRBYyneZ
+w5Pz+XWQvL+74JW6LsWNc2EF0xCEqLOJuC9zjPAqbr7uroNLghGxYf13YdqbG5oj
+/4x+ogEG3dF/U5YIwVr658DKyESMV6eoYV9mDVfTuJastkqcwero+5ZAKfYVMLUE
+sMwFtoTDJFmVf6JlkOWwsxp1WcQ/MRQK1cyqOoUFUgYylgdh3yeCDPeF22Ax8AlQ
+xbcaI+GwfQL1FB7Jy+h+KjME9lE/UpgV6Qt2R1xNSmvFCBWu+NFX6epwFP/JRbkM
+fLz0beYFUvmMgLtwVpEPSwIDAQABo4IDeTCCA3UwHwYDVR0jBBgwFoAUPdNQpdag
+re7zSmAKZdMh1Pj41g8wHQYDVR0OBBYEFMnCU2FmnV+rJfQmzQ84mqhJ6kipMCUG
+A1UdEQQeMByCCmdpdGh1Yi5jb22CDnd3dy5naXRodWIuY29tMA4GA1UdDwEB/wQE
+AwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0fBG4wbDA0
+oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItZXYtc2VydmVyLWcy
+LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTItZXYtc2Vy
+dmVyLWcyLmNybDBLBgNVHSAERDBCMDcGCWCGSAGG/WwCATAqMCgGCCsGAQUFBwIB
+FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAcGBWeBDAEBMIGIBggrBgEF
+BQcBAQR8MHowJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBS
+BggrBgEFBQcwAoZGaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0
+U0hBMkV4dGVuZGVkVmFsaWRhdGlvblNlcnZlckNBLmNydDAMBgNVHRMBAf8EAjAA
+MIIBfgYKKwYBBAHWeQIEAgSCAW4EggFqAWgAdgCkuQmQtBhYFIe7E6LMZ3AKPDWY
+BPkb37jjd80OyA3cEAAAAWNBYm0KAAAEAwBHMEUCIQDRZp38cTWsWH2GdBpe/uPT
+Wnsu/m4BEC2+dIcvSykZYgIgCP5gGv6yzaazxBK2NwGdmmyuEFNSg2pARbMJlUFg
+U5UAdgBWFAaaL9fC7NP14b1Esj7HRna5vJkRXMDvlJhV1onQ3QAAAWNBYm0tAAAE
+AwBHMEUCIQCi7omUvYLm0b2LobtEeRAYnlIo7n6JxbYdrtYdmPUWJQIgVgw1AZ51
+vK9ENinBg22FPxb82TvNDO05T17hxXRC2IYAdgC72d+8H4pxtZOUI5eqkntHOFeV
+CqtS6BqQlmQ2jh7RhQAAAWNBYm3fAAAEAwBHMEUCIQChzdTKUU2N+XcqcK0OJYrN
+8EYynloVxho4yPk6Dq3EPgIgdNH5u8rC3UcslQV4B9o0a0w204omDREGKTVuEpxG
+eOQwDQYJKoZIhvcNAQELBQADggEBAHAPWpanWOW/ip2oJ5grAH8mqQfaunuCVE+v
+ac+88lkDK/LVdFgl2B6kIHZiYClzKtfczG93hWvKbST4NRNHP9LiaQqdNC17e5vN
+HnXVUGw+yxyjMLGqkgepOnZ2Rb14kcTOGp4i5AuJuuaMwXmCo7jUwPwfLe1NUlVB
+Kqg6LK0Hcq4K0sZnxE8HFxiZ92WpV2AVWjRMEc/2z2shNoDvxvFUYyY1Oe67xINk
+myQKc+ygSBZzyLnXSFVWmHr3u5dcaaQGGAR42v6Ydr4iL38Hd4dOiBma+FXsXBIq
+WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik=
+-----END CERTIFICATE-----
+"#).expect("Failed to parse cert");
+        x.valid_names.sort();
+        assert_eq!(x, Certificate {
+            valid_names: vec!["github.com".into(), "www.github.com".into()],
+            valid_ipaddrs: vec![],
+        });
+    }
+
+    #[test]
+    fn test_parse_pem_1_1_1_1() {
+        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+MIID9DCCA3qgAwIBAgIQBWzetBRl/ycHFsBukRYuGTAKBggqhkjOPQQDAjBMMQsw
+CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMSYwJAYDVQQDEx1EaWdp
+Q2VydCBFQ0MgU2VjdXJlIFNlcnZlciBDQTAeFw0xODAzMzAwMDAwMDBaFw0yMDAz
+MjUxMjAwMDBaMGwxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMN
+U2FuIEZyYW5jaXNjbzEZMBcGA1UEChMQQ2xvdWRmbGFyZSwgSW5jLjEdMBsGA1UE
+AwwUKi5jbG91ZGZsYXJlLWRucy5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC
+AASyRQsxrFBjziHmfDQjGsXBU0WWl3oxh7vg6h2V9f8lBMp18PY/td9R6VvJPa20
+AwVzIJI+dL6OSxviaIZEbmK7o4ICHDCCAhgwHwYDVR0jBBgwFoAUo53mH/naOU/A
+buiRy5Wl2jHiCp8wHQYDVR0OBBYEFN+XTeVDs7BBp0LykM+Jf64SV4ThMGMGA1Ud
+EQRcMFqCFCouY2xvdWRmbGFyZS1kbnMuY29thwQBAQEBhwQBAAABghJjbG91ZGZs
+YXJlLWRucy5jb22HECYGRwBHAAAAAAAAAAAAERGHECYGRwBHAAAAAAAAAAAAEAEw
+DgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBp
+BgNVHR8EYjBgMC6gLKAqhihodHRwOi8vY3JsMy5kaWdpY2VydC5jb20vc3NjYS1l
+Y2MtZzEuY3JsMC6gLKAqhihodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vc3NjYS1l
+Y2MtZzEuY3JsMEwGA1UdIARFMEMwNwYJYIZIAYb9bAEBMCowKAYIKwYBBQUHAgEW
+HGh0dHBzOi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwCAYGZ4EMAQICMHsGCCsGAQUF
+BwEBBG8wbTAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQuY29tMEUG
+CCsGAQUFBzAChjlodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGlnaUNlcnRF
+Q0NTZWN1cmVTZXJ2ZXJDQS5jcnQwDAYDVR0TAQH/BAIwADAKBggqhkjOPQQDAgNo
+ADBlAjEAjoyy2Ogh1i1/Kh9+psMc1OChlQIvQF6AkojZS8yliar6m8q5nqC3qe0h
+HR0fExwLAjAueWRnHX4QJ9loqMhsPk3NB0Cs0mStsNDNG6/DpCYw7XmjoG3y1LS7
+ZkZZmqNn2Q8=
+-----END CERTIFICATE-----
+"#).expect("Failed to parse cert");
+        x.valid_names.sort();
+        x.valid_ipaddrs.sort();
+        assert_eq!(x, Certificate {
+            valid_names: vec![
+                "*.cloudflare-dns.com".into(),
+                "cloudflare-dns.com".into(),
+            ],
+            valid_ipaddrs: vec![
+                "1.0.0.1".parse().unwrap(),
+                "1.1.1.1".parse().unwrap(),
+                "2606:4700:4700::1001".parse().unwrap(),
+                "2606:4700:4700::1111".parse().unwrap(),
+            ],
+        });
+    }
+
+    #[test]
+    fn test_san_extension() {
+        let (rem, ext) = san_extension(&[48, 28,
+                130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109,
+                130, 14, 119, 119, 119, 46, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109
+            ])
+            .expect("Failed to parse extension");
+        assert!(rem.is_empty());
+        assert_eq!(ext, vec![
+            AlternativeName::DnsName(String::from("github.com")),
+            AlternativeName::DnsName(String::from("www.github.com")),
+        ]);
+    }
+
+    #[test]
+    fn test_san_value_dns() {
+        let (rem, v) = san_value(&[130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109])
+            .expect("Failed to parse san value");
+        let v = v.expect("Extension contains invalid data");
+        assert!(rem.is_empty());
+        assert_eq!(v, AlternativeName::DnsName(String::from("github.com")));
+    }
+
+    #[test]
+    fn test_san_value_ipaddr() {
+        let (rem, v) = san_value(&[135, 4, 1, 1, 1, 1])
+            .expect("Failed to parse san value");
+        let v = v.expect("Extension contains invalid data");
+        assert!(rem.is_empty());
+        assert_eq!(v, AlternativeName::IpAddr("1.1.1.1".parse().unwrap()));
+    }
+}

src/db.rs

@@ -7,22 +7,63 @@ use diesel::sql_types::Bool;
 use diesel::prelude::*;
 use models::*;
 use schema::*;
+use std::str::FromStr;
 use paths;
 use migrations;
 use worker;
+use workspaces::Workspace;
 
 
+#[derive(Debug)]
+pub enum DbChange {
+    Insert,
+    Update(Update),
+    None,
+}
+
+impl DbChange {
+    pub fn is_some(&self) -> bool {
+        match self {
+            DbChange::None => false,
+            _ => true,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum Family {
+    Domain,
+    Subdomain,
+    IpAddr,
+    SubdomainIpAddr,
+    Url,
+    Email,
+}
+
+impl FromStr for Family {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Family> {
+        Ok(match s {
+            "domain" => Family::Domain,
+            "subdomain" => Family::Subdomain,
+            "ipaddr" => Family::IpAddr,
+            "subdomain-ipaddr" => Family::SubdomainIpAddr,
+            "url" => Family::Url,
+            "email" => Family::Email,
+            _ => bail!("Unknown object family"),
+        })
+    }
+}
+
 pub struct Database {
-    name: String,
+    name: Workspace,
     db: SqliteConnection,
 }
 
 impl Database {
-    pub fn establish<I: Into<String>>(name: I) -> Result<Database> {
-        // TODO: enforce safe name for database
-        let name = name.into();
-
-        let path = paths::data_dir()?.join(name.clone() + ".db");
+    pub fn establish(name: Workspace) -> Result<Database> {
+        let path = paths::data_dir()?.join(name.to_string() + ".db");
         let path = path.into_os_string().into_string()
             .map_err(|_| format_err!("Failed to convert db path to utf-8"))?;
 
@@ -31,6 +72,8 @@ impl Database {
                 .context("Failed to connect to database")?;
             migrations::run(&db)
                 .context("Failed to run migrations")?;
+            db.execute("PRAGMA foreign_keys = ON")
+                .context("Failed to enforce foreign keys")?;
             Ok(db)
         }, false)?;
 
@@ -49,128 +92,141 @@ impl Database {
     }
 
     /// Returns true if we didn't have this value yet
-    pub fn insert_generic(&self, object: &Object) -> Result<(bool, i32)> {
+    pub fn insert_generic(&self, object: &Insert) -> Result<Option<(DbChange, i32)>> {
         match object {
-            Object::Subdomain(object) => self.insert_subdomain_struct(&NewSubdomain {
+            Insert::Domain(object) => self.insert_struct(NewDomain {
+                value: &object.value,
+            }),
+            Insert::Subdomain(object) => self.insert_struct(NewSubdomain {
                 domain_id: object.domain_id,
                 value: &object.value,
+                resolvable: object.resolvable,
             }),
-            Object::IpAddr(object) => self.insert_ipaddr_struct(&NewIpAddr {
+            Insert::IpAddr(object) => self.insert_struct(NewIpAddr {
                 family: &object.family,
                 value: &object.value,
+                continent: object.continent.as_ref(),
+                continent_code: object.continent_code.as_ref(),
+                country: object.country.as_ref(),
+                country_code: object.country_code.as_ref(),
+                city: object.city.as_ref(),
+                longitude: object.longitude,
+                latitude: object.latitude,
+                asn: object.asn,
+                as_org: object.as_org.as_ref(),
             }),
-            Object::SubdomainIpAddr(object) => self.insert_subdomain_ipaddr_struct(&NewSubdomainIpAddr {
+            Insert::SubdomainIpAddr(object) => self.insert_subdomain_ipaddr_struct(&NewSubdomainIpAddr {
                 subdomain_id: object.subdomain_id,
                 ip_addr_id: object.ip_addr_id,
             }),
-            Object::Url(object) => self.insert_url_struct(&NewUrl {
+            Insert::Url(object) => self.insert_struct(NewUrl {
                 subdomain_id: object.subdomain_id,
                 value: &object.value,
                 status: object.status,
-                body: object.body.as_ref().map(|x| x.as_ref()),
+                body: object.body.as_ref(),
+                online: object.online,
+                title: object.title.as_ref(),
+                redirect: object.redirect.as_ref(),
+            }),
+            Insert::Email(object) => self.insert_struct(NewEmail {
+                value: &object.value,
+                valid: object.valid,
             }),
         }
     }
 
-    pub fn insert_domain(&self, domain: &str) -> Result<()> {
-        let new_domain = NewDomain {
-            value: domain,
-        };
-
-        diesel::insert_into(domains::table)
-            .values(&new_domain)
-            .execute(&self.db)?;
-
-        Ok(())
-    }
-
     /// Returns true if we didn't have this value yet
-    pub fn insert_subdomain(&self, subdomain: &str, domain: &str) -> Result<(bool, i32)> {
-        let domain_id = match Domain::id_opt(self, domain)? {
-            Some(domain_id) => domain_id,
-            None => {
-                self.insert_domain(domain)?;
-                Domain::id(self, domain)?
-            },
-        };
+    pub fn insert_struct<T: InsertableStruct<M>, M: Model + Scopable>(&self, obj: T) -> Result<Option<(DbChange, i32)>> {
+        if let Some(existing) = M::get_opt(self, obj.value())? {
+            // entity is out of scope
+            if !existing.scoped() {
+                return Ok(None);
+            }
 
-        let new_subdomain = NewSubdomain {
-            domain_id,
-            value: &subdomain,
-        };
-
-        self.insert_subdomain_struct(&new_subdomain)
-    }
-
-    /// Returns true if we didn't have this value yet
-    pub fn insert_subdomain_struct(&self, subdomain: &NewSubdomain) -> Result<(bool, i32)> {
-        // upsert is not supported by diesel
-
-        if let Some(subdomain_id) = Subdomain::id_opt(self, subdomain.value)? {
-            // TODO: right now we don't have any fields to update
-            Ok((false, subdomain_id))
+            let update = obj.upsert(&existing);
+            if update.is_dirty() {
+                update.apply(&self)?;
+                Ok(Some((DbChange::Update(update.generic()), existing.id())))
+            } else {
+                Ok(Some((DbChange::None, existing.id())))
+            }
         } else {
-            diesel::insert_into(subdomains::table)
-                .values(subdomain)
-                .execute(&self.db)?;
-            let id = Subdomain::id(self, subdomain.value)?;
-            Ok((true, id))
+            obj.insert(&self)?;
+            let id = M::get_id(self, obj.value())?;
+            Ok(Some((DbChange::Insert, id)))
         }
     }
 
-    pub fn insert_ipaddr(&self, family: &str, ipaddr: &str) -> Result<(bool, i32)> {
-        // TODO: maybe check if valid
-        let new_ipaddr = NewIpAddr {
-            family: &family,
-            value: &ipaddr,
-        };
-
-        self.insert_ipaddr_struct(&new_ipaddr)
-    }
-
-    pub fn insert_ipaddr_struct(&self, ipaddr: &NewIpAddr) -> Result<(bool, i32)> {
-        // upsert is not supported by diesel
-
-        if let Some(ipaddr_id) = IpAddr::id_opt(self, ipaddr.value)? {
-            // TODO: right now we don't have any fields to update
-            Ok((false, ipaddr_id))
-        } else {
-            diesel::insert_into(ipaddrs::table)
-                .values(ipaddr)
-                .execute(&self.db)?;
-            let id = IpAddr::id(self, ipaddr.value)?;
-            Ok((true, id))
-        }
-    }
-
-    pub fn insert_subdomain_ipaddr(&self, subdomain_id: i32, ip_addr_id: i32) -> Result<(bool, i32)> {
-        self.insert_subdomain_ipaddr_struct(&NewSubdomainIpAddr {
-            subdomain_id,
-            ip_addr_id,
-        })
-    }
-
-    pub fn insert_subdomain_ipaddr_struct(&self, subdomain_ipaddr: &NewSubdomainIpAddr) -> Result<(bool, i32)> {
-        if let Some(subdomain_ipaddr_id) = SubdomainIpAddr::id_opt(self, &(subdomain_ipaddr.subdomain_id, subdomain_ipaddr.ip_addr_id))? {
-            Ok((false, subdomain_ipaddr_id))
+    pub fn insert_subdomain_ipaddr_struct(&self, subdomain_ipaddr: &NewSubdomainIpAddr) -> Result<Option<(DbChange, i32)>> {
+        if let Some(subdomain_ipaddr_id) = SubdomainIpAddr::get_id_opt(self, &(subdomain_ipaddr.subdomain_id, subdomain_ipaddr.ip_addr_id))? {
+            Ok(Some((DbChange::None, subdomain_ipaddr_id)))
         } else {
             diesel::insert_into(subdomain_ipaddrs::table)
                 .values(subdomain_ipaddr)
                 .execute(&self.db)?;
-            let id = SubdomainIpAddr::id(self, &(subdomain_ipaddr.subdomain_id, subdomain_ipaddr.ip_addr_id))?;
-            Ok((true, id))
+            let id = SubdomainIpAddr::get_id(self, &(subdomain_ipaddr.subdomain_id, subdomain_ipaddr.ip_addr_id))?;
+            Ok(Some((DbChange::Insert, id)))
         }
     }
 
-    pub fn insert_url_struct(&self, url: &NewUrl) -> Result<(bool, i32)> {
-        if let Some(url_id) = Url::id_opt(self, url.value)? {
-            Ok((false, url_id))
-        } else {
-            diesel::insert_into(urls::table)
-                .values(url)
-                .execute(&self.db)?;
-            let id = Url::id(self, url.value)?;
-            Ok((true, id))
+    //
+
+    pub fn update_generic(&self, object: &Update) -> Result<i32> {
+        match object {
+            Update::Subdomain(object) => self.update_subdomain(object),
+            Update::IpAddr(object) => self.update_ipaddr(object),
+            Update::Url(object) => self.update_url(object),
+            Update::Email(object) => self.update_email(object),
+        }
+    }
+
+    pub fn update_subdomain(&self, subdomain: &SubdomainUpdate) -> Result<i32> {
+        use schema::subdomains::columns::*;
+        diesel::update(subdomains::table.filter(id.eq(subdomain.id)))
+            .set(subdomain)
+            .execute(&self.db)?;
+        Ok(subdomain.id)
+    }
+
+    pub fn update_ipaddr(&self, ipaddr: &IpAddrUpdate) -> Result<i32> {
+        use schema::ipaddrs::columns::*;
+        diesel::update(ipaddrs::table.filter(id.eq(ipaddr.id)))
+            .set(ipaddr)
+            .execute(&self.db)?;
+        Ok(ipaddr.id)
+    }
+
+    pub fn update_url(&self, url: &UrlUpdate) -> Result<i32> {
+        use schema::urls::columns::*;
+        diesel::update(urls::table.filter(id.eq(url.id)))
+            .set(url)
+            .execute(&self.db)?;
+        Ok(url.id)
+    }
+
+    pub fn update_email(&self, email: &EmailUpdate) -> Result<i32> {
+        use schema::emails::columns::*;
+        diesel::update(emails::table.filter(id.eq(email.id)))
+            .set(email)
+            .execute(&self.db)?;
+        Ok(email.id)
+    }
+
+    fn get_opt_typed<T: Model + Scopable>(&self, value: &T::ID) -> Result<Option<i32>> {
+        match T::get_opt(self, &value)? {
+            Some(ref obj) if obj.scoped() => Ok(Some(obj.id())),
+            _ => Ok(None),
+        }
+    }
+
+    pub fn get_opt(&self, family: &Family, value: &str) -> Result<Option<i32>> {
+        match family {
+            Family::Domain => self.get_opt_typed::<Domain>(&value),
+            Family::Subdomain => self.get_opt_typed::<Subdomain>(&value),
+            Family::IpAddr => self.get_opt_typed::<IpAddr>(&value),
+            Family::SubdomainIpAddr => bail!("Unsupported operation"),
+            Family::Url => self.get_opt_typed::<Url>(&value),
+            Family::Email => self.get_opt_typed::<Email>(&value),
         }
     }
 
@@ -183,6 +239,18 @@ impl Database {
     pub fn filter<T: Model>(&self, filter: &Filter) -> Result<Vec<T>> {
         T::filter(self, filter)
     }
+
+    pub fn scope<T: Scopable>(&self, filter: &Filter) -> Result<usize> {
+        T::scope(self, filter)
+    }
+
+    pub fn noscope<T: Scopable>(&self, filter: &Filter) -> Result<usize> {
+        T::noscope(self, filter)
+    }
+
+    pub fn delete<T: Model>(&self, filter: &Filter) -> Result<usize> {
+        T::delete(self, filter)
+    }
 }
 
 #[derive(Debug, PartialEq)]
@@ -197,11 +265,23 @@ impl Filter {
         }
     }
 
+    fn escape(value: &str) -> String {
+        let mut out = String::from("'");
+        for c in value.chars() {
+            match c {
+                '\'' => out.push_str("''"),
+                c => out.push(c),
+            }
+        }
+        out.push('\'');
+        out
+    }
+
     pub fn parse(mut args: &[String]) -> Result<Filter> {
         debug!("Parsing query: {:?}", args);
 
         if args.is_empty() {
-            return Ok(Filter::new("1"));
+            bail!("Filter condition is required");
         }
 
         if args[0].to_lowercase() == "where" {
@@ -215,16 +295,17 @@ impl Filter {
         let mut expect_value = false;
 
         for arg in args {
-            if let Some(idx) = arg.find("=") {
+            if let Some(idx) = arg.find('=') {
                 if idx != 0 {
                     let (key, value) = arg.split_at(idx);
-                    query += &format!(" {} = {:?}", key, &value[1..]);
+                    query += &format!(" {} = {}", key, Self::escape(&value[1..]));
                     continue;
                 }
             }
 
             if expect_value {
-                query += &format!(" {:?}", arg);
+                query.push(' ');
+                query.push_str(&Self::escape(arg));
                 expect_value = false;
             } else {
                 if ["=", "!=", "like"].contains(&arg.to_lowercase().as_str()) {
@@ -239,10 +320,26 @@ impl Filter {
         Ok(Filter::new(query))
     }
 
+    pub fn parse_optional(args: &[String]) -> Result<Filter> {
+        debug!("Parsing optional query: {:?}", args);
+
+        if args.is_empty() {
+            debug!("Using filter with no condition");
+            return Ok(Filter::new("1"));
+        }
+
+        Self::parse(args)
+    }
+
     pub fn query(&self) -> &str {
         &self.query
     }
 
+    pub fn and_scoped(&self) -> Filter {
+        let query = format!("({}) AND unscoped=0", self.query);
+        Filter::new(query)
+    }
+
     pub fn sql(&self) -> SqlLiteral<Bool> {
         sql::<Bool>(&self.query)
     }
@@ -258,7 +355,7 @@ mod tests {
         let filter = Filter::parse(&["where".to_string(),
                                      "value=1".to_string(),
                                     ]).unwrap();
-        assert_eq!(filter, Filter::new(" value = \"1\""));
+        assert_eq!(filter, Filter::new(" value = '1'"));
     }
 
     #[test]
@@ -266,7 +363,7 @@ mod tests {
         let filter = Filter::parse(&["where".to_string(),
                                      "value=abc".to_string(),
                                     ]).unwrap();
-        assert_eq!(filter, Filter::new(" value = \"abc\""));
+        assert_eq!(filter, Filter::new(" value = 'abc'"));
     }
 
     #[test]
@@ -276,7 +373,7 @@ mod tests {
                                      "=".to_string(),
                                      "asdf".to_string(),
                                     ]).unwrap();
-        assert_eq!(filter, Filter::new(" value = \"asdf\""));
+        assert_eq!(filter, Filter::new(" value = 'asdf'"));
     }
 
     #[test]
@@ -290,7 +387,7 @@ mod tests {
                                      "=".to_string(),
                                      "1".to_string(),
                                     ]).unwrap();
-        assert_eq!(filter, Filter::new(" value = \"foobar\" and id = \"1\""));
+        assert_eq!(filter, Filter::new(" value = 'foobar' and id = '1'"));
     }
 
     #[test]
@@ -300,6 +397,42 @@ mod tests {
                                      "like".to_string(),
                                      "%foobar".to_string(),
                                     ]).unwrap();
-        assert_eq!(filter, Filter::new(" value like \"%foobar\""));
+        assert_eq!(filter, Filter::new(" value like '%foobar'"));
+    }
+
+    #[test]
+    fn test_filter_backslash1() {
+        let filter = Filter::parse(&["where".to_string(),
+                                     "value=\\".to_string(),
+                                    ]).unwrap();
+        assert_eq!(filter, Filter::new(" value = '\\'"));
+    }
+
+    #[test]
+    fn test_filter_backslash2() {
+        let filter = Filter::parse(&["where".to_string(),
+                                     "value".to_string(),
+                                     "=".to_string(),
+                                     "\\".to_string(),
+                                    ]).unwrap();
+        assert_eq!(filter, Filter::new(" value = '\\'"));
+    }
+
+    #[test]
+    fn test_filter_quote1() {
+        let filter = Filter::parse(&["where".to_string(),
+                                     "value=a'b".to_string(),
+                                    ]).unwrap();
+        assert_eq!(filter, Filter::new(" value = 'a''b'"));
+    }
+
+    #[test]
+    fn test_filter_quote2() {
+        let filter = Filter::parse(&["where".to_string(),
+                                     "value".to_string(),
+                                     "=".to_string(),
+                                     "a'b".to_string(),
+                                    ]).unwrap();
+        assert_eq!(filter, Filter::new(" value = 'a''b'"));
     }
 }

src/engine/ctx.rs

@@ -1,8 +1,11 @@
 use errors::*;
 
-use engine::Reporter;
+use db::Family;
+use engine::{Environment, Reporter};
+use geoip::{GeoIP, AsnDB};
 use hlua::{self, AnyLuaValue};
-use models::Object;
+use models::{Insert, Update};
+use psl::Psl;
 use runtime;
 use serde_json;
 use std::collections::HashMap;
@@ -10,7 +13,7 @@ use std::result;
 use std::sync::{Arc, Mutex};
 use chrootable_https::dns::DnsConfig;
 use web::{HttpSession, HttpRequest, RequestOptions};
-use worker::Event;
+use worker::{Event, LogEvent, DatabaseEvent};
 
 
 pub trait State {
@@ -27,23 +30,49 @@ pub trait State {
     fn recv(&self) -> Result<serde_json::Value>;
 
     fn info(&self, msg: String) {
-        self.send(&Event::Info(msg))
+        self.send(&Event::Log(LogEvent::Info(msg)))
+    }
+
+    fn error(&self, msg: String) {
+        self.send(&Event::Log(LogEvent::Error(msg)))
     }
 
     fn status(&self, msg: String) {
-        self.send(&Event::Status(msg))
+        self.send(&Event::Log(LogEvent::Status(msg)))
     }
 
-    fn db_insert(&self, object: Object) -> Result<i32> {
-        self.send(&Event::Object(object));
+    fn db_insert(&self, object: Insert) -> Result<Option<i32>> {
+        self.send(&Event::Database(DatabaseEvent::Insert(object)));
         let reply = self.recv()?;
-        let reply: result::Result<i32, String> = serde_json::from_value(reply)?;
+        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
 
         reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
     }
 
+    fn db_select(&self, family: Family, value: String) -> Result<Option<i32>> {
+        self.send(&Event::Database(DatabaseEvent::Select((family, value))));
+        let reply = self.recv()?;
+        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
+
+        reply.map_err(|err| format_err!("Failed to query database: {:?}", err))
+    }
+
+    fn db_update(&self, object: String, update: Update) -> Result<Option<i32>> {
+        self.send(&Event::Database(DatabaseEvent::Update((object, update))));
+        let reply = self.recv()?;
+        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
+
+        reply.map_err(|err| format_err!("Failed to update database: {:?}", err))
+    }
+
     fn dns_config(&self) -> Arc<DnsConfig>;
 
+    fn psl(&self) -> Arc<Psl>;
+
+    fn geoip(&self) -> Arc<GeoIP>;
+
+    fn asn(&self) -> Arc<AsnDB>;
+
     fn http_mksession(&self) -> String;
 
     fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest;
@@ -51,12 +80,15 @@ pub trait State {
     fn register_in_jar(&self, session: &str, key: String, value: String);
 }
 
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone)]
 pub struct LuaState {
     error: Arc<Mutex<Option<Error>>>,
     logger: Arc<Mutex<Option<Arc<Mutex<Box<Reporter>>>>>>,
     http_sessions: Arc<Mutex<HashMap<String, HttpSession>>>,
     dns_config: Arc<DnsConfig>,
+    psl: Arc<Psl>,
+    geoip: Arc<GeoIP>,
+    asn: Arc<AsnDB>,
 }
 
 impl State for LuaState {
@@ -104,6 +136,18 @@ impl State for LuaState {
         self.dns_config.clone()
     }
 
+    fn psl(&self) -> Arc<Psl> {
+        self.psl.clone()
+    }
+
+    fn geoip(&self) -> Arc<GeoIP> {
+        self.geoip.clone()
+    }
+
+    fn asn(&self) -> Arc<AsnDB> {
+        self.asn.clone()
+    }
+
     fn http_mksession(&self) -> String {
         let mut mtx = self.http_sessions.lock().unwrap();
         let (id, session) = HttpSession::new();
@@ -131,19 +175,29 @@ pub struct Script {
     code: String,
 }
 
-fn ctx<'a>(dns_config: DnsConfig) -> (hlua::Lua<'a>, Arc<LuaState>) {
+fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
     debug!("Creating lua context");
     let mut lua = hlua::Lua::new();
     lua.open_string();
     let state = Arc::new(LuaState {
-        dns_config: Arc::new(dns_config),
-        ..Default::default()
+        error: Arc::new(Mutex::new(None)),
+        logger: Arc::new(Mutex::new(None)),
+        http_sessions: Arc::new(Mutex::new(HashMap::new())),
+
+        dns_config: Arc::new(env.dns_config),
+        psl: Arc::new(env.psl),
+        geoip: Arc::new(env.geoip),
+        asn: Arc::new(env.asn),
     });
 
     runtime::clear_err(&mut lua, state.clone());
     runtime::db_add(&mut lua, state.clone());
+    runtime::db_select(&mut lua, state.clone());
+    runtime::db_update(&mut lua, state.clone());
     runtime::dns(&mut lua, state.clone());
     runtime::error(&mut lua, state.clone());
+    runtime::asn_lookup(&mut lua, state.clone());
+    runtime::geoip_lookup(&mut lua, state.clone());
     runtime::html_select(&mut lua, state.clone());
     runtime::html_select_list(&mut lua, state.clone());
     runtime::http_mksession(&mut lua, state.clone());
@@ -151,14 +205,21 @@ fn ctx<'a>(dns_config: DnsConfig) -> (hlua::Lua<'a>, Arc<LuaState>) {
     runtime::http_send(&mut lua, state.clone());
     runtime::info(&mut lua, state.clone());
     runtime::json_decode(&mut lua, state.clone());
-    runtime::json_encode(&mut lua, state.clone());
     runtime::json_decode_stream(&mut lua, state.clone());
+    runtime::json_encode(&mut lua, state.clone());
     runtime::last_err(&mut lua, state.clone());
+    runtime::pgp_pubkey(&mut lua, state.clone());
+    runtime::pgp_pubkey_armored(&mut lua, state.clone());
     runtime::print(&mut lua, state.clone());
+    runtime::psl_domain_from_dns_name(&mut lua, state.clone());
+    runtime::regex_find(&mut lua, state.clone());
+    runtime::regex_find_all(&mut lua, state.clone());
     runtime::sleep(&mut lua, state.clone());
     runtime::status(&mut lua, state.clone());
     runtime::url_join(&mut lua, state.clone());
     runtime::url_parse(&mut lua, state.clone());
+    runtime::utf8_decode(&mut lua, state.clone());
+    runtime::x509_parse_pem(&mut lua, state.clone());
 
     debug!("Created lua context");
 
@@ -185,8 +246,11 @@ impl Script {
         })
     }
 
-    pub fn run(&self, dns_config: DnsConfig, tx: Arc<Mutex<Box<Reporter>>>, arg: AnyLuaValue) -> Result<()> {
-        let (mut lua, state) = ctx(dns_config);
+    pub fn run(&self, env: Environment,
+                      tx: Arc<Mutex<Box<Reporter>>>,
+                      arg: AnyLuaValue
+    ) -> Result<()> {
+        let (mut lua, state) = ctx(env);
 
         debug!("Initializing lua module");
         lua.execute::<()>(&self.code)?;
@@ -217,7 +281,22 @@ impl Script {
     #[cfg(test)]
     pub fn test(&self) -> Result<()> {
         use engine::tests::DummyReporter;
+        use geoip::Maxmind;
         let dns_config = DnsConfig::from_system()?;
-        self.run(dns_config, DummyReporter::new(), AnyLuaValue::LuaNil)
+        let psl = Psl::from_str(r#"
+// ===BEGIN ICANN DOMAINS===
+com
+// ===END ICANN DOMAINS===
+"#)?;
+        let geoip = GeoIP::open_or_download()?;
+        let asn = AsnDB::open_or_download()?;
+
+        let env = Environment {
+            dns_config,
+            psl,
+            geoip,
+            asn,
+        };
+        self.run(env, DummyReporter::new(), AnyLuaValue::LuaNil)
     }
 }

src/engine/isolation.rs

@@ -1,13 +1,14 @@
 use errors::*;
-use channel;
 use chrootable_https::dns::DnsConfig;
-use engine::{Module, Event, Reporter};
+use engine::{Environment, Module, Reporter};
+use geoip::{GeoIP, AsnDB};
+use psl::Psl;
 use serde_json;
+use worker::{Event, Event2, LogEvent, DatabaseEvent, ExitEvent, EventSender};
 
 use std::env;
 use std::io::prelude::*;
 use std::io::{self, BufReader, BufRead, Stdin, Stdout};
-use std::result;
 use std::sync::{mpsc, Arc, Mutex};
 use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
 
@@ -21,6 +22,7 @@ pub struct StartCommand {
 
 impl StartCommand {
     pub fn new(dns_config: DnsConfig, module: Module, arg: serde_json::Value) -> StartCommand {
+        // TODO: compress psl
         StartCommand {
             dns_config,
             module,
@@ -61,12 +63,12 @@ impl Supervisor {
 
     pub fn send_start(&mut self, dns_config: DnsConfig, module: Module, arg: serde_json::Value) -> Result<()> {
         let start = serde_json::to_value(StartCommand::new(dns_config, module, arg))?;
-        self.send(start)?;
+        self.send(&start)?;
         Ok(())
     }
 
-    pub fn send(&mut self, value: serde_json::Value) -> Result<()> {
-        let mut value = serde_json::to_string(&value)?;
+    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
+        let mut value = serde_json::to_string(value)?;
         value.push('\n');
         self.stdin.write_all(value.as_bytes())?;
         debug!("Supervisor sent: {:?}", value);
@@ -92,6 +94,17 @@ impl Supervisor {
             bail!("Child signaled error")
         }
     }
+
+    pub fn send_event_callback(&mut self, event: DatabaseEvent, tx: &EventSender) {
+        let (tx2, rx2) = mpsc::channel();
+        tx.send(Event2::Database((event, tx2)));
+        let reply = rx2.recv().unwrap();
+
+        let value = serde_json::to_value(reply).expect("Failed to serialize reply");
+        if let Err(_) = self.send(&value) {
+            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -137,7 +150,7 @@ impl Reporter for StdioReporter {
     }
 }
 
-pub fn spawn_module(module: Module, tx: channel::Sender<(Event, Option<mpsc::Sender<result::Result<i32, String>>>)>, arg: serde_json::Value) -> Result<()> {
+pub fn spawn_module(module: Module, tx: &EventSender, arg: serde_json::Value) -> Result<()> {
     let dns_config = DnsConfig::from_system()?;
 
     let mut supervisor = Supervisor::setup(&module)?;
@@ -145,22 +158,14 @@ pub fn spawn_module(module: Module, tx: channel::Sender<(Event, Option<mpsc::Sen
 
     loop {
         match supervisor.recv()? {
-            Event::Done => break,
-            Event::Error(err) => {
-                tx.send((Event::Error(err), None));
+            Event::Log(event) => tx.send(Event2::Log(event)),
+            Event::Database(object) => supervisor.send_event_callback(object, &tx),
+            Event::Exit(event) => {
+                if let ExitEvent::Err(err) = event {
+                    tx.send(Event2::Log(LogEvent::Error(err)));
+                }
                 break;
             },
-            Event::Object(object) => {
-                let (tx2, rx2) = mpsc::channel();
-                tx.send((Event::Object(object), Some(tx2)));
-                let reply = rx2.recv().unwrap();
-
-                let value = serde_json::to_value(reply).expect("Failed to serialize reply");
-                if let Err(_) = supervisor.send(value) {
-                    tx.send((Event::Error("Failed to send to child".into()), None));
-                }
-            },
-            event => tx.send((event, None)),
         }
     }
 
@@ -169,20 +174,32 @@ pub fn spawn_module(module: Module, tx: channel::Sender<(Event, Option<mpsc::Sen
     Ok(())
 }
 
-pub fn run_worker() -> Result<()> {
+pub fn run_worker(geoip: GeoIP, asn: AsnDB, psl: String) -> Result<()> {
     let mut reporter = StdioReporter::setup();
     let start = reporter.recv_start()?;
 
+    let psl = Psl::from_str(&psl)
+        .context("Failed to load public suffix list")?;
+
+    let environment = Environment {
+        dns_config: start.dns_config,
+        psl,
+        geoip,
+        asn,
+    };
+
     let mtx: Arc<Mutex<Box<Reporter>>> = Arc::new(Mutex::new(Box::new(reporter)));
-    let result = start.module.run(start.dns_config, mtx.clone(), start.arg.into());
+    let result = start.module.run(environment,
+                                  mtx.clone(),
+                                  start.arg.into());
     let mut reporter = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
                         .into_inner().expect("Failed to consume Mutex");
 
-    if let Err(err) = result {
-        reporter.send(&Event::Error(err.to_string()))?;
-    } else {
-        reporter.send(&Event::Done)?;
-    }
+    let event = match result {
+        Ok(_) => ExitEvent::Ok,
+        Err(err) => ExitEvent::Err(err.to_string()),
+    };
+    reporter.send(&Event::Exit(event))?;
 
     Ok(())
 }

src/engine/mod.rs

@@ -1,5 +1,6 @@
 use errors::*;
 
+use geoip::{GeoIP, AsnDB};
 use json::LuaJsonValue;
 use serde_json;
 use std::fs;
@@ -8,9 +9,12 @@ use std::path::PathBuf;
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex};
 use engine::ctx::Script;
+use sn0int_common::ModuleID;
 use sn0int_common::metadata::{Metadata, Source};
 use chrootable_https::dns::DnsConfig;
+use psl::Psl;
 use paths;
+use std::cmp::Ordering;
 use term;
 use worker::{self, Event};
 
@@ -19,6 +23,15 @@ pub mod isolation;
 pub mod structs;
 
 
+/// Data that is passed to every script
+#[derive(Debug)]
+pub struct Environment {
+  pub dns_config: DnsConfig,
+  pub psl: Psl,
+  pub geoip: GeoIP,
+  pub asn: AsnDB,
+}
+
 #[derive(Debug)]
 pub struct Engine {
     path: PathBuf,
@@ -39,13 +52,13 @@ impl Engine {
         Ok(engine)
     }
 
-    pub fn reload_modules(&mut self) -> Result<()> {
+    pub fn reload_modules(&mut self) -> Result<usize> {
         let modules = worker::spawn_fn("Loading modules", || {
             self.reload_modules_quiet()?;
             Ok(self.list().len())
         }, true)?;
         term::info(&format!("Loaded {} modules", modules));
-        Ok(())
+        Ok(modules)
     }
 
     pub fn reload_modules_quiet(&mut self) -> Result<()> {
@@ -99,12 +112,13 @@ impl Engine {
         }
     }
 
-    // TODO: this should return an iter
     pub fn list(&self) -> Vec<&Module> {
-        self.modules.iter()
+        let mut modules: Vec<_> = self.modules.iter()
             .filter(|(key, _)| key.contains('/'))
             .flat_map(|(_, v)| v.iter())
-            .collect()
+            .collect();
+        modules.sort_by(|a, b| a.cmp_canonical(b));
+        modules
     }
 
     pub fn variants(&self) -> Vec<String> {
@@ -154,6 +168,13 @@ impl Module {
         format!("{}/{}", self.author, self.name)
     }
 
+    pub fn id(&self) -> ModuleID {
+        ModuleID {
+            author: self.author.to_string(),
+            name: self.name.to_string(),
+        }
+    }
+
     pub fn description(&self) -> &str {
         &self.description
     }
@@ -166,9 +187,17 @@ impl Module {
         &self.source
     }
 
-    pub fn run(&self, dns_config: DnsConfig, reporter: Arc<Mutex<Box<Reporter>>>, arg: LuaJsonValue) -> Result<()> {
+    pub fn run(&self, env: Environment, reporter: Arc<Mutex<Box<Reporter>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Executing lua script {}", self.canonical());
-        self.script.run(dns_config, reporter, arg.into())
+        self.script.run(env, reporter, arg.into())
+    }
+
+    fn cmp_canonical(&self, other: &Module) -> Ordering {
+        if self.author == other.author {
+            self.name.cmp(&other.name)
+        } else {
+            self.author.cmp(&other.author)
+        }
     }
 }
 

src/engine/structs.rs

@@ -1,5 +1,17 @@
+use errors::*;
 use hlua::{AnyHashableLuaValue, AnyLuaValue};
 use std::collections::{self, HashMap};
+use json::LuaJsonValue;
+use serde;
+use serde_json;
+
+
+pub fn from_lua<T>(x: LuaJsonValue) -> Result<T>
+    where for<'de> T: serde::Deserialize<'de>
+{
+    serde_json::from_value(x.into())
+        .map_err(Error::from)
+}
 
 #[derive(Debug, Default)]
 pub struct LuaMap(HashMap<AnyHashableLuaValue, AnyLuaValue>);
@@ -91,3 +103,51 @@ impl Into<AnyLuaValue> for LuaMap {
         )
     }
 }
+
+#[derive(Debug, Default)]
+pub struct LuaList(Vec<(AnyLuaValue, AnyLuaValue)>);
+
+impl LuaList {
+    #[inline]
+    pub fn new() -> LuaList {
+        LuaList::default()
+    }
+
+    pub fn push<V: Into<AnyLuaValue>>(&mut self, v: V) {
+        let idx = self.0.len() + 1;
+        self.0.push((AnyLuaValue::LuaNumber(idx as f64), v.into()));
+    }
+
+    pub fn push_str<I: Into<String>>(&mut self, v: I) {
+        self.push(AnyLuaValue::LuaString(v.into()))
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+}
+
+impl Into<AnyLuaValue> for LuaList {
+    fn into(self: LuaList) -> AnyLuaValue {
+        AnyLuaValue::LuaArray(self.0)
+    }
+}
+
+pub fn byte_array(bytes: AnyLuaValue) -> Result<Vec<u8>> {
+    match bytes {
+        AnyLuaValue::LuaAnyString(bytes) => Ok(bytes.0),
+        AnyLuaValue::LuaString(bytes) => Ok(bytes.into_bytes()),
+        AnyLuaValue::LuaArray(bytes) => {
+            Ok(bytes.into_iter()
+                .map(|num| match num.1 {
+                    AnyLuaValue::LuaNumber(num) if num <= 255.0 && num >= 0.0 && (num % 1.0 == 0.0) =>
+                            Ok(num as u8),
+                    AnyLuaValue::LuaNumber(num) =>
+                            Err(format_err!("number is out of range: {:?}", num)),
+                    _ => Err(format_err!("unexpected type: {:?}", num)),
+                })
+                .collect::<Result<_>>()?)
+        },
+        _ => Err(format_err!("invalid type: {:?}", bytes)),
+    }
+}

src/geoip/mod.rs

@@ -0,0 +1,169 @@
+use errors::*;
+use archive;
+use chrootable_https::Client;
+use maxminddb::{self, geoip2};
+use std::fmt;
+use std::fs::File;
+use std::net::IpAddr;
+use std::path::Path;
+use paths;
+use worker;
+
+pub static GEOIP_CITY_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz";
+pub static GEOIP_ASN_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz";
+
+pub mod models;
+use self::models::GeoLookup;
+use self::models::AsnLookup;
+
+
+pub trait Maxmind: Sized {
+    fn archive_filename() -> &'static str;
+
+    fn archive_url() -> &'static str;
+
+    fn new(reader: maxminddb::Reader) -> Self;
+
+    // TODO: refactor this to return Path
+    fn cache_path() -> Result<String> {
+        // use system path if exists
+        let path = Path::new("/usr/share/GeoIP/");
+        let path = path.join(Self::archive_filename());
+
+        if path.exists() {
+            let path = path.to_str()
+                .ok_or_else(|| format_err!("Failed to decode path"))?;
+            return Ok(path.to_string());
+        }
+
+        // use cache path
+        let path = paths::cache_dir()?
+            .join(Self::archive_filename());
+        let path = path.to_str()
+            .ok_or_else(|| format_err!("Failed to decode path"))?;
+        Ok(path.to_string())
+    }
+
+    fn open(path: &str) -> Result<Self> {
+        let reader = maxminddb::Reader::open(path)
+            .context("Failed to open geoip database")?;
+        Ok(Self::new(reader))
+    }
+
+    fn open_or_download() -> Result<Self> {
+        let path = Self::cache_path()?;
+
+        if File::open(&path).is_err() {
+            worker::spawn_fn(&format!("Downloading {:?}", Self::archive_filename()), || {
+                Self::download(&path, Self::archive_filename(), Self::archive_url())
+            }, false)?;
+        };
+
+        Self::open(&path)
+    }
+
+    fn download<P: AsRef<Path>>(path: P, filter: &str, url: &str) -> Result<()> {
+        debug!("Downloading {:?}...", url);
+        let client = Client::with_system_resolver()?;
+        let resp = client.get(url)?;
+        debug!("Downloaded {} bytes", resp.body.len());
+        archive::extract(&mut &resp.body[..], filter, path)?;
+        Ok(())
+    }
+}
+
+pub struct GeoIP {
+    reader: maxminddb::Reader,
+}
+
+impl fmt::Debug for GeoIP {
+    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        write!(w, "GeoIP {{ ... }}")
+    }
+}
+
+impl Maxmind for GeoIP {
+    #[inline]
+    fn archive_filename() -> &'static str {
+        "GeoLite2-City.mmdb"
+    }
+
+    #[inline]
+    fn archive_url() -> &'static str {
+        GEOIP_CITY_URL
+    }
+
+    #[inline]
+    fn new(reader: maxminddb::Reader) -> Self {
+        GeoIP {
+            reader
+        }
+    }
+}
+
+impl GeoIP {
+    pub fn lookup(&self, ip: IpAddr) -> Result<GeoLookup> {
+        let city: geoip2::City = self.reader.lookup(ip)?;
+        debug!("GeoIP result: {:?}", city);
+        Ok(GeoLookup::from(city))
+    }
+}
+
+pub struct AsnDB {
+    reader: maxminddb::Reader,
+}
+
+impl fmt::Debug for AsnDB {
+    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        write!(w, "GeoIP {{ ... }}")
+    }
+}
+
+impl Maxmind for AsnDB {
+    #[inline]
+    fn archive_filename() -> &'static str {
+        "GeoLite2-ASN.mmdb"
+    }
+
+    #[inline]
+    fn archive_url() -> &'static str {
+        GEOIP_ASN_URL
+    }
+
+    #[inline]
+    fn new(reader: maxminddb::Reader) -> Self {
+        AsnDB {
+            reader
+        }
+    }
+}
+
+impl AsnDB {
+    pub fn lookup(&self, ip: IpAddr) -> Result<AsnLookup> {
+        let isp: geoip2::Isp = self.reader.lookup(ip)?;
+        debug!("ASN result: {:?}", isp);
+        AsnLookup::try_from(isp)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_geoip_lookup() {
+        let ip = "1.1.1.1".parse().unwrap();
+        let geoip = GeoIP::open_or_download().expect("Failed to load geoip");
+        let lookup = geoip.lookup(ip).expect("GeoIP lookup failed");
+        println!("{:#?}", lookup);
+        assert_eq!(lookup.city, None);
+    }
+
+    #[test]
+    fn test_asn_lookup() {
+        let ip = "1.1.1.1".parse().unwrap();
+        let asndb = AsnDB::open_or_download().expect("Failed to load asndb");
+        let lookup = asndb.lookup(ip).expect("ASN lookup failed");
+        println!("{:#?}", lookup);
+    }
+}

src/geoip/models.rs

@@ -0,0 +1,170 @@
+use errors::*;
+use maxminddb::geoip2;
+use std::collections::BTreeMap;
+
+
+fn from_geoip_model_names(names: Option<BTreeMap<String, String>>) -> Option<String> {
+    let names = match names {
+        Some(names) => names,
+        _ => return None,
+    };
+
+    names.get("en")
+        .map(|x| x.to_owned())
+}
+
+#[derive(Debug, Serialize)]
+pub struct GeoLookup {
+    pub continent: Option<String>,
+    pub continent_code: Option<String>,
+    pub country: Option<String>,
+    pub country_code: Option<String>,
+    pub city: Option<String>,
+    pub latitude: Option<f64>,
+    pub longitude: Option<f64>,
+}
+
+impl From<geoip2::City> for GeoLookup {
+    fn from(lookup: geoip2::City) -> GeoLookup {
+        // parse maxminddb lookup
+        let continent = match lookup.continent {
+            Some(continent) => Continent::from_maxmind(continent),
+            _ => None,
+        };
+        let country = match lookup.country {
+            Some(country) => Country::from_maxmind(country),
+            _ => None,
+        };
+        let city = match lookup.city {
+            Some(city) => from_geoip_model_names(city.names),
+            _ => None,
+        };
+        let location = match lookup.location {
+            Some(location) => Location::from_maxmind(&location),
+            _ => None,
+        };
+
+        // flatten datastructure
+        let (continent, continent_code) = match continent {
+            Some(x) => (Some(x.name), Some(x.code)),
+            _ => (None, None),
+        };
+        let (country, country_code) = match country {
+            Some(x) => (Some(x.name), Some(x.code)),
+            _ => (None, None),
+        };
+        let (latitude, longitude) = match location {
+            Some(x) => (Some(x.latitude), Some(x.longitude)),
+            _ => (None, None),
+        };
+
+        // return result
+        GeoLookup {
+            continent,
+            continent_code,
+            country,
+            country_code,
+            city,
+            latitude,
+            longitude,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct Continent {
+    code: String,
+    name: String,
+}
+
+impl Continent {
+    fn from_maxmind(continent: geoip2::model::Continent) -> Option<Self> {
+        let code = match continent.code {
+            Some(code) => code,
+            _ => return None,
+        };
+        let name = match from_geoip_model_names(continent.names) {
+            Some(name) => name,
+            _ => return None,
+        };
+
+        Some(Continent {
+            code,
+            name,
+        })
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct Country {
+    code: String,
+    name: String,
+}
+
+impl Country {
+    fn from_maxmind(country: geoip2::model::Country) -> Option<Self> {
+        let code = match country.iso_code {
+            Some(code) => code,
+            _ => return None,
+        };
+        let name = match from_geoip_model_names(country.names) {
+            Some(name) => name,
+            _ => return None,
+        };
+
+        Some(Country {
+            code,
+            name,
+        })
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct Location {
+    latitude: f64,
+    longitude: f64,
+}
+
+impl Location {
+    fn from_maxmind(location: &geoip2::model::Location) -> Option<Self> {
+        let latitude = match location.latitude {
+            Some(latitude) => latitude,
+            _ => return None,
+        };
+        let longitude = match location.longitude {
+            Some(longitude) => longitude,
+            _ => return None,
+        };
+
+        Some(Location {
+            latitude,
+            longitude,
+        })
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct AsnLookup {
+    asn: u32,
+    as_org: String,
+}
+
+impl AsnLookup {
+    pub fn try_from(lookup: geoip2::Isp) -> Result<AsnLookup> {
+        // parse maxminddb lookup
+        let asn = match lookup.autonomous_system_number {
+            Some(asn) => asn,
+            _ => bail!("autonomous_system_number not set"),
+        };
+        let as_org = match lookup.autonomous_system_organization {
+            Some(org) => org,
+            _ => bail!("autonomous_system_organization not set"),
+        };
+
+        // return result
+        Ok(AsnLookup {
+            asn,
+            as_org,
+        })
+    }
+}

src/json.rs

@@ -64,6 +64,7 @@ impl Into<AnyLuaValue> for LuaJsonValue {
             // TODO: not sure if this might fail
             LuaJsonValue::Number(v) => AnyLuaValue::LuaNumber(v.as_f64().unwrap()),
             LuaJsonValue::String(v) => AnyLuaValue::LuaString(v),
+            // TODO: ensure lua tables always start at 1
             LuaJsonValue::Array(v) => AnyLuaValue::LuaArray(v.into_iter().enumerate()
                 .map(|(i, x)| (AnyLuaValue::LuaNumber(i as f64), x.into()))
                 .collect()

src/lib.rs

@@ -12,15 +12,31 @@ extern crate publicsuffix;
 extern crate chrootable_https;
 extern crate trust_dns_proto;
 extern crate url;
+#[cfg(target_os = "linux")]
 extern crate nix;
+#[cfg(target_os = "linux")]
 extern crate caps;
+#[cfg(target_os = "linux")]
 extern crate syscallz;
+#[cfg(target_os = "openbsd")]
+#[macro_use] extern crate pledge;
+#[cfg(target_os = "openbsd")]
+extern crate unveil;
 extern crate hlua_badtouch as hlua;
 extern crate base64;
 extern crate kuchiki;
 extern crate ctrlc;
 extern crate opener;
 extern crate separator;
+extern crate sloppy_rfc4880;
+extern crate regex;
+extern crate toml;
+extern crate maxminddb;
+extern crate tar;
+extern crate libflate;
+extern crate threadpool;
+extern crate x509_parser;
+extern crate der_parser;
 extern crate serde;
 extern crate serde_json;
 extern crate serde_urlencoded;
@@ -31,15 +47,20 @@ extern crate serde_urlencoded;
 #[macro_use] extern crate diesel;
 #[macro_use] extern crate diesel_migrations;
 #[macro_use] extern crate crossbeam_channel as channel;
+#[macro_use] extern crate nom;
 
 pub mod api;
+pub mod archive;
 pub mod args;
 pub mod auth;
 pub mod cmd;
 pub mod complete;
+pub mod config;
+pub mod crt;
 pub mod db;
 pub mod errors;
 pub mod engine;
+pub mod geoip;
 pub mod html;
 pub mod json;
 pub mod migrations;
@@ -54,5 +75,6 @@ pub mod runtime;
 pub mod term;
 pub mod web;
 pub mod worker;
+pub mod workspaces;
 pub mod psl;
 pub mod utils;

src/main.rs

@@ -4,11 +4,16 @@ extern crate env_logger;
 #[macro_use] extern crate failure;
 extern crate structopt;
 
+use env_logger::Env;
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
 use sn0int::cmd;
+use sn0int::complete;
+use sn0int::config::Config;
 use sn0int::errors::*;
 use sn0int::engine::{self, Module};
+use sn0int::geoip::{GeoIP, AsnDB, Maxmind};
+use sn0int::psl::Psl;
 use sn0int::registry;
 use sn0int::sandbox;
 use sn0int::shell;
@@ -16,8 +21,8 @@ use structopt::StructOpt;
 use std::path::Path;
 
 
-fn run_run(gargs: &Args, args: &args::Run) -> Result<()> {
-    let mut rl = shell::init(gargs)?;
+fn run_run(gargs: &Args, args: &args::Run, config: Config) -> Result<()> {
+    let mut rl = shell::init(gargs, config)?;
 
     if let Some(module) = &args.module {
         let module = rl.engine().get(&module)?.clone();
@@ -37,13 +42,21 @@ fn run_run(gargs: &Args, args: &args::Run) -> Result<()> {
         bail!("At least one module or file need to be provided");
     }
 
-    cmd::run_cmd::execute(&mut rl)
+    cmd::run_cmd::execute(&mut rl, args.threads)
 }
 
 fn run_sandbox() -> Result<()> {
+    // TODO: this file should be processed after the sandbox is up
+    let path = GeoIP::cache_path()?;
+    let geoip = GeoIP::open(&path)?;
+
+    let path = AsnDB::cache_path()?;
+    let asn = AsnDB::open(&path)?;
+    let psl = Psl::open_into_string()?;
+
     sandbox::init()
         .context("Failed to init sandbox")?;
-    engine::isolation::run_worker()
+    engine::isolation::run_worker(geoip, asn, psl)
 }
 
 fn run() -> Result<()> {
@@ -53,19 +66,24 @@ fn run() -> Result<()> {
         sandbox::fasten_seatbelt()?;
     }
 
+    let config = Config::load_or_default()
+        .context("Failed to load config")?;
+
     match args.subcommand {
-        Some(SubCommand::Run(ref run)) => run_run(&args, run),
+        Some(SubCommand::Run(ref run)) => run_run(&args, run, config),
         Some(SubCommand::Sandbox(_)) => run_sandbox(),
-        Some(SubCommand::Login(_)) => auth::run_login(),
-        Some(SubCommand::Publish(ref publish)) => registry::run_publish(&args, publish),
-        Some(SubCommand::Install(ref install)) => registry::run_install(install),
-        Some(SubCommand::Search(ref search)) => registry::run_search(search),
-        None => shell::run(&args),
+        Some(SubCommand::Login(_)) => auth::run_login(&config),
+        Some(SubCommand::Publish(ref publish)) => registry::run_publish(&args, publish, &config),
+        Some(SubCommand::Install(ref install)) => registry::run_install(install, &config),
+        Some(SubCommand::Search(ref search)) => registry::run_search(search, &config),
+        Some(SubCommand::Completions(ref completions)) => complete::run_generate(completions),
+        None => shell::run(&args, config),
     }
 }
 
 fn main() {
-    env_logger::init();
+    env_logger::init_from_env(Env::default()
+        .default_filter_or("off"));
 
     if let Err(err) = run() {
         eprintln!("Error: {}", err);

src/models/domain.rs

@@ -1,24 +1,24 @@
 use errors::*;
+use diesel;
 use diesel::prelude::*;
 use models::*;
 
 
-#[derive(Identifiable, Queryable, Serialize, PartialEq, Debug)]
+#[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="domains"]
 pub struct Domain {
     pub id: i32,
     pub value: String,
-}
-
-impl fmt::Display for Domain {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.value)
-    }
+    pub unscoped: bool,
 }
 
 impl Model for Domain {
     type ID = str;
 
+    fn to_string(&self) -> String {
+        self.value.to_owned()
+    }
+
     fn list(db: &Database) -> Result<Vec<Self>> {
         use schema::domains::dsl::*;
 
@@ -36,6 +36,22 @@ impl Model for Domain {
         Ok(results)
     }
 
+    fn delete(db: &Database, filter: &Filter) -> Result<usize> {
+        use schema::domains::dsl::*;
+
+        diesel::delete(domains.filter(filter.sql()))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
+    fn id(&self) -> i32 {
+        self.id
+    }
+
+    fn value(&self) -> &Self::ID {
+        &self.value
+    }
+
     fn by_id(db: &Database, my_id: i32) -> Result<Self> {
         use schema::domains::dsl::*;
 
@@ -45,25 +61,47 @@ impl Model for Domain {
         Ok(domain)
     }
 
-    fn id(db: &Database, query: &Self::ID) -> Result<i32> {
+    fn get(db: &Database, query: &Self::ID) -> Result<Self> {
         use schema::domains::dsl::*;
 
-        let domain_id = domains.filter(value.eq(query))
-            .select(id)
-            .first::<i32>(db.db())?;
+        let domain = domains.filter(value.eq(query))
+            .first::<Self>(db.db())?;
 
-        Ok(domain_id)
+        Ok(domain)
     }
 
-    fn id_opt(db: &Database, query: &Self::ID) -> Result<Option<i32>> {
+    fn get_opt(db: &Database, query: &Self::ID) -> Result<Option<Self>> {
         use schema::domains::dsl::*;
 
-        let domain_id = domains.filter(value.eq(query))
-            .select(id)
-            .first::<i32>(db.db())
+        let domain = domains.filter(value.eq(query))
+            .first::<Self>(db.db())
             .optional()?;
 
-        Ok(domain_id)
+        Ok(domain)
+    }
+}
+
+impl Scopable for Domain {
+    fn scoped(&self) -> bool {
+        !self.unscoped
+    }
+
+    fn scope(db: &Database, filter: &Filter) -> Result<usize> {
+        use schema::domains::dsl::*;
+
+        diesel::update(domains.filter(filter.sql()))
+            .set(unscoped.eq(false))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
+    fn noscope(db: &Database, filter: &Filter) -> Result<usize> {
+        use schema::domains::dsl::*;
+
+        diesel::update(domains.filter(filter.sql()))
+            .set(unscoped.eq(true))
+            .execute(db.db())
+            .map_err(Error::from)
     }
 }
 
@@ -88,11 +126,16 @@ impl Printable<PrintableDomain> for Domain {
 pub struct DetailedDomain {
     id: i32,
     value: String,
+    unscoped: bool,
 }
 
 impl fmt::Display for DetailedDomain {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{:?}\x1b[0m", self.id, self.value)
+        if !self.unscoped {
+            write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{:?}\x1b[0m", self.id, self.value)
+        } else {
+            write!(w, "\x1b[90m#{}, {:?}\x1b[0m", self.id, self.value)
+        }
     }
 }
 
@@ -103,6 +146,7 @@ impl Detailed for Domain {
         Ok(DetailedDomain {
             id: self.id,
             value: self.value.to_string(),
+            unscoped: self.unscoped,
         })
     }
 }
@@ -112,3 +156,40 @@ impl Detailed for Domain {
 pub struct NewDomain<'a> {
     pub value: &'a str,
 }
+
+impl<'a> InsertableStruct<Domain> for NewDomain<'a> {
+    fn value(&self) -> &str {
+        self.value
+    }
+
+    fn insert(&self, db: &Database) -> Result<()> {
+        diesel::insert_into(domains::table)
+            .values(self)
+            .execute(db.db())?;
+        Ok(())
+    }
+}
+
+impl<'a> Upsertable<Domain> for NewDomain<'a> {
+    type Update = NullUpdate;
+
+    fn upsert(self, existing: &Domain) -> Self::Update {
+        Self::Update {
+            id: existing.id,
+        }
+    }
+}
+
+#[derive(Debug, Insertable, Serialize, Deserialize)]
+#[table_name="domains"]
+pub struct NewDomainOwned {
+    pub value: String,
+}
+
+impl Printable<PrintableDomain> for NewDomainOwned {
+    fn printable(&self, _db: &Database) -> Result<PrintableDomain> {
+        Ok(PrintableDomain {
+            value: self.value.to_string(),
+        })
+    }
+}

src/models/email.rs

@@ -0,0 +1,230 @@
+use errors::*;
+use diesel;
+use diesel::prelude::*;
+use models::*;
+
+
+#[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
+#[table_name="emails"]
+pub struct Email {
+    pub id: i32,
+    pub value: String,
+    pub unscoped: bool,
+    pub valid: Option<bool>,
+}
+
+impl Model for Email {
+    type ID = str;
+
+    fn to_string(&self) -> String {
+        self.value.to_owned()
+    }
+
+    fn list(db: &Database) -> Result<Vec<Self>> {
+        use schema::emails::dsl::*;
+
+        let results = emails.load::<Self>(db.db())?;
+
+        Ok(results)
+    }
+
+    fn filter(db: &Database, filter: &Filter) -> Result<Vec<Self>> {
+        use schema::emails::dsl::*;
+
+        let query = emails.filter(filter.sql());
+        let results = query.load::<Self>(db.db())?;
+
+        Ok(results)
+    }
+
+    fn delete(db: &Database, filter: &Filter) -> Result<usize> {
+        use schema::emails::dsl::*;
+
+        diesel::delete(emails.filter(filter.sql()))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
+    fn id(&self) -> i32 {
+        self.id
+    }
+
+    fn value(&self) -> &Self::ID {
+        &self.value
+    }
+
+    fn by_id(db: &Database, my_id: i32) -> Result<Self> {
+        use schema::emails::dsl::*;
+
+        let domain = emails.filter(id.eq(my_id))
+            .first::<Self>(db.db())?;
+
+        Ok(domain)
+    }
+
+    fn get(db: &Database, query: &Self::ID) -> Result<Self> {
+        use schema::emails::dsl::*;
+
+        let email = emails.filter(value.eq(query))
+            .first::<Self>(db.db())?;
+
+        Ok(email)
+    }
+
+    fn get_opt(db: &Database, query: &Self::ID) -> Result<Option<Self>> {
+        use schema::emails::dsl::*;
+
+        let email = emails.filter(value.eq(query))
+            .first::<Self>(db.db())
+            .optional()?;
+
+        Ok(email)
+    }
+}
+
+impl Scopable for Email {
+    fn scoped(&self) -> bool {
+        !self.unscoped
+    }
+
+    fn scope(db: &Database, filter: &Filter) -> Result<usize> {
+        use schema::emails::dsl::*;
+
+        diesel::update(emails.filter(filter.sql()))
+            .set(unscoped.eq(false))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
+    fn noscope(db: &Database, filter: &Filter) -> Result<usize> {
+        use schema::emails::dsl::*;
+
+        diesel::update(emails.filter(filter.sql()))
+            .set(unscoped.eq(true))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+}
+
+pub struct PrintableEmail {
+    value: String,
+}
+
+impl fmt::Display for PrintableEmail {
+    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        write!(w, "{:?}", self.value)
+    }
+}
+
+impl Printable<PrintableEmail> for Email {
+    fn printable(&self, _db: &Database) -> Result<PrintableEmail> {
+        Ok(PrintableEmail {
+            value: self.value.to_string(),
+        })
+    }
+}
+
+pub struct DetailedEmail {
+    id: i32,
+    value: String,
+    unscoped: bool,
+}
+
+impl fmt::Display for DetailedEmail {
+    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        if !self.unscoped {
+            write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{:?}\x1b[0m", self.id, self.value)
+        } else {
+            write!(w, "\x1b[90m#{}, {:?}\x1b[0m", self.id, self.value)
+        }
+    }
+}
+
+impl Detailed for Email {
+    type T = DetailedEmail;
+
+    fn detailed(&self, _db: &Database) -> Result<Self::T> {
+        Ok(DetailedEmail {
+            id: self.id,
+            value: self.value.to_string(),
+            unscoped: self.unscoped,
+        })
+    }
+}
+
+#[derive(Insertable)]
+#[table_name="emails"]
+pub struct NewEmail<'a> {
+    pub value: &'a str,
+    pub valid: Option<bool>,
+}
+
+impl<'a> InsertableStruct<Email> for NewEmail<'a> {
+    fn value(&self) -> &str {
+        self.value
+    }
+
+    fn insert(&self, db: &Database) -> Result<()> {
+        diesel::insert_into(emails::table)
+            .values(self)
+            .execute(db.db())?;
+        Ok(())
+    }
+}
+
+impl<'a> Upsertable<Email> for NewEmail<'a> {
+    type Update = EmailUpdate;
+
+    fn upsert(self, existing: &Email) -> Self::Update {
+        Self::Update {
+            id: existing.id,
+            valid: Self::upsert_opt(self.valid, &existing.valid),
+        }
+    }
+}
+
+#[derive(Debug, Insertable, Serialize, Deserialize)]
+#[table_name="emails"]
+pub struct NewEmailOwned {
+    pub value: String,
+    pub valid: Option<bool>,
+}
+
+impl Printable<PrintableEmail> for NewEmailOwned {
+    fn printable(&self, _db: &Database) -> Result<PrintableEmail> {
+        Ok(PrintableEmail {
+            value: self.value.to_string(),
+        })
+    }
+}
+
+#[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
+#[table_name="emails"]
+pub struct EmailUpdate {
+    pub id: i32,
+    pub valid: Option<bool>,
+}
+
+impl Upsert for EmailUpdate {
+    fn is_dirty(&self) -> bool {
+        self.valid.is_some()
+    }
+
+    fn generic(self) -> Update {
+        Update::Email(self)
+    }
+
+    fn apply(&self, db: &Database) -> Result<i32> {
+        db.update_email(self)
+    }
+}
+
+impl Updateable<Email> for EmailUpdate {
+    fn changeset(&mut self, existing: &Email) {
+        Self::clear_if_equal(&mut self.valid, &existing.valid);
+    }
+
+    fn fmt(&self, updates: &mut Vec<String>) {
+        Self::push_value(updates, "valid", &self.valid);
+    }
+}

src/models/ipaddr.rs

@@ -1,23 +1,36 @@
 use errors::*;
+use diesel;
 use diesel::prelude::*;
-use json::LuaJsonValue;
 use models::*;
-use serde_json;
 use std::net;
 use std::result;
 
 
-#[derive(Identifiable, Queryable, Associations, PartialEq, Debug)]
+#[derive(Identifiable, Queryable, Associations, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="ipaddrs"]
 pub struct IpAddr {
     pub id: i32,
     pub family: String,
     pub value: String,
+    pub unscoped: bool,
+    pub continent: Option<String>,
+    pub continent_code: Option<String>,
+    pub country: Option<String>,
+    pub country_code: Option<String>,
+    pub city: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+    pub asn: Option<i32>,
+    pub as_org: Option<String>,
 }
 
 impl Model for IpAddr {
     type ID = str;
 
+    fn to_string(&self) -> String {
+        self.value.to_owned()
+    }
+
     fn list(db: &Database) -> Result<Vec<Self>> {
         use schema::ipaddrs::dsl::*;
 
@@ -35,6 +48,22 @@ impl Model for IpAddr {
         Ok(results)
     }
 
+    fn delete(db: &Database, filter: &Filter) -> Result<usize> {
+        use schema::ipaddrs::dsl::*;
+
+        diesel::delete(ipaddrs.filter(filter.sql()))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
+    fn id(&self) -> i32 {
+        self.id
+    }
+
+    fn value(&self) -> &Self::ID {
+        &self.value
+    }
+
     fn by_id(db: &Database, my_id: i32) -> Result<Self> {
         use schema::ipaddrs::dsl::*;
 
@@ -44,25 +73,47 @@ impl Model for IpAddr {
         Ok(ipaddr)
     }
 
-    fn id(db: &Database, query: &Self::ID) -> Result<i32> {
+    fn get(db: &Database, query: &Self::ID) -> Result<Self> {
         use schema::ipaddrs::dsl::*;
 
-        let ipaddr_id = ipaddrs.filter(value.eq(query))
-            .select(id)
-            .first::<i32>(db.db())?;
+        let ipaddr = ipaddrs.filter(value.eq(query))
+            .first::<Self>(db.db())?;
 
-        Ok(ipaddr_id)
+        Ok(ipaddr)
     }
 
-    fn id_opt(db: &Database, query: &Self::ID) -> Result<Option<i32>> {
+    fn get_opt(db: &Database, query: &Self::ID) -> Result<Option<Self>> {
         use schema::ipaddrs::dsl::*;
 
-        let ipaddr_id = ipaddrs.filter(value.eq(query))
-            .select(id)
-            .first::<i32>(db.db())
+        let ipaddr = ipaddrs.filter(value.eq(query))
+            .first::<Self>(db.db())
             .optional()?;
 
-        Ok(ipaddr_id)
+        Ok(ipaddr)
+    }
+}
+
+impl Scopable for IpAddr {
+    fn scoped(&self) -> bool {
+        !self.unscoped
+    }
+
+    fn scope(db: &Database, filter: &Filter) -> Result<usize> {
+        use schema::ipaddrs::dsl::*;
+
+        diesel::update(ipaddrs.filter(filter.sql()))
+            .set(unscoped.eq(false))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
+    fn noscope(db: &Database, filter: &Filter) -> Result<usize> {
+        use schema::ipaddrs::dsl::*;
+
+        diesel::update(ipaddrs.filter(filter.sql()))
+            .set(unscoped.eq(true))
+            .execute(db.db())
+            .map_err(Error::from)
     }
 }
 
@@ -104,14 +155,78 @@ pub struct DetailedIpAddr {
     id: i32,
     value: net::IpAddr,
     subdomains: Vec<PrintableSubdomain>,
+    unscoped: bool,
+    continent: Option<String>,
+    country: Option<String>,
+    city: Option<String>,
+    asn: Option<i32>,
+    as_org: Option<String>,
 }
 
 impl fmt::Display for DetailedIpAddr {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{}\x1b[0m", self.id, self.value)?;
+        if !self.unscoped {
+            write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{}\x1b[0m", self.id, self.value)?;
 
-        for subdomain in &self.subdomains {
-            write!(w, "\n\t\x1b[33m{}\x1b[0m", subdomain)?;
+            if let Some(ref continent) = self.continent {
+                write!(w, " [{}", continent)?;
+
+                if let Some(ref country) = self.country {
+                    write!(w, " / {}", country)?;
+                }
+
+                if let Some(ref city) = self.city {
+                    write!(w, " / {}", city)?;
+                }
+
+                write!(w, "]")?;
+            }
+
+            if let Some(ref asn) = self.asn {
+                write!(w, " [{}", asn)?;
+
+                if let Some(ref as_org) = self.as_org {
+                    write!(w, " / {:?}", as_org)?;
+                }
+
+                write!(w, "]")?;
+            }
+
+            for subdomain in &self.subdomains {
+                write!(w, "\n\t\x1b[33m{}\x1b[0m", subdomain)?;
+            }
+        } else {
+            write!(w, "\x1b[90m#{}, {}", self.id, self.value)?;
+
+            if let Some(ref continent) = self.continent {
+                write!(w, " [{}", continent)?;
+
+                if let Some(ref country) = self.country {
+                    write!(w, " / {}", country)?;
+                }
+
+                if let Some(ref city) = self.city {
+                    write!(w, " / {}", city)?;
+                }
+
+                write!(w, "]")?;
+            }
+
+            if let Some(ref asn) = self.asn {
+                write!(w, " [{}", asn)?;
+
+                if let Some(ref as_org) = self.as_org {
+                    write!(w, " / {:?}", as_org)?;
+                }
+
+                write!(w, "]")?;
+            }
+
+            write!(w, "\x1b[0m");
+
+            for subdomain in &self.subdomains {
+                write!(w, "\n\t\x1b[90m{}\x1b[0m", subdomain)?;
+            }
         }
 
         Ok(())
@@ -130,6 +245,12 @@ impl Detailed for IpAddr {
             id: self.id,
             value: self.value.parse()?,
             subdomains,
+            unscoped: self.unscoped,
+            continent: self.continent.clone(),
+            country: self.country.clone(),
+            city: self.city.clone(),
+            asn: self.asn,
+            as_org: self.as_org.clone(),
         })
     }
 }
@@ -139,6 +260,47 @@ impl Detailed for IpAddr {
 pub struct NewIpAddr<'a> {
     pub family: &'a str,
     pub value: &'a str,
+    pub continent: Option<&'a String>,
+    pub continent_code: Option<&'a String>,
+    pub country: Option<&'a String>,
+    pub country_code: Option<&'a String>,
+    pub city: Option<&'a String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+    pub asn: Option<i32>,
+    pub as_org: Option<&'a String>,
+}
+
+impl<'a> InsertableStruct<IpAddr> for NewIpAddr<'a> {
+    fn value(&self) -> &str {
+        self.value
+    }
+
+    fn insert(&self, db: &Database) -> Result<()> {
+        diesel::insert_into(ipaddrs::table)
+            .values(self)
+            .execute(db.db())?;
+        Ok(())
+    }
+}
+
+impl<'a> Upsertable<IpAddr> for NewIpAddr<'a> {
+    type Update = IpAddrUpdate;
+
+    fn upsert(self, existing: &IpAddr) -> Self::Update {
+        Self::Update {
+            id: existing.id,
+            continent: Self::upsert_str(self.continent, &existing.continent),
+            continent_code: Self::upsert_str(self.continent_code, &existing.continent_code),
+            country: Self::upsert_str(self.country, &existing.country),
+            country_code: Self::upsert_str(self.country_code, &existing.country_code),
+            city: Self::upsert_str(self.city, &existing.city),
+            latitude: Self::upsert_opt(self.latitude, &existing.latitude),
+            longitude: Self::upsert_opt(self.longitude, &existing.longitude),
+            asn: Self::upsert_opt(self.asn, &existing.asn),
+            as_org: Self::upsert_str(self.as_org, &existing.as_org),
+        }
+    }
 }
 
 #[derive(Debug, Insertable, Serialize, Deserialize)]
@@ -146,13 +308,15 @@ pub struct NewIpAddr<'a> {
 pub struct NewIpAddrOwned {
     pub family: String,
     pub value: String,
-}
-
-impl NewIpAddrOwned {
-    pub fn from_lua(x: LuaJsonValue) -> Result<NewIpAddrOwned> {
-        let x = serde_json::from_value(x.into())?;
-        Ok(x)
-    }
+    pub continent: Option<String>,
+    pub continent_code: Option<String>,
+    pub country: Option<String>,
+    pub country_code: Option<String>,
+    pub city: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+    pub asn: Option<i32>,
+    pub as_org: Option<String>,
 }
 
 impl Printable<PrintableIpAddr> for NewIpAddrOwned {
@@ -162,3 +326,66 @@ impl Printable<PrintableIpAddr> for NewIpAddrOwned {
         })
     }
 }
+
+#[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
+#[table_name="ipaddrs"]
+pub struct IpAddrUpdate {
+    pub id: i32,
+    pub continent: Option<String>,
+    pub continent_code: Option<String>,
+    pub country: Option<String>,
+    pub country_code: Option<String>,
+    pub city: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+    pub asn: Option<i32>,
+    pub as_org: Option<String>,
+}
+
+impl Upsert for IpAddrUpdate {
+    fn is_dirty(&self) -> bool {
+        self.continent.is_some() ||
+        self.continent_code.is_some() ||
+        self.country.is_some() ||
+        self.country_code.is_some() ||
+        self.city.is_some() ||
+        self.latitude.is_some() ||
+        self.longitude.is_some() ||
+        self.asn.is_some() ||
+        self.as_org.is_some()
+    }
+
+    fn generic(self) -> Update {
+        Update::IpAddr(self)
+    }
+
+    fn apply(&self, db: &Database) -> Result<i32> {
+        db.update_ipaddr(self)
+    }
+}
+
+impl Updateable<IpAddr> for IpAddrUpdate {
+    fn changeset(&mut self, existing: &IpAddr) {
+        Self::clear_if_equal(&mut self.continent, &existing.continent);
+        Self::clear_if_equal(&mut self.continent_code, &existing.continent_code);
+        Self::clear_if_equal(&mut self.country, &existing.country);
+        Self::clear_if_equal(&mut self.country_code, &existing.country_code);
+        Self::clear_if_equal(&mut self.city, &existing.city);
+        Self::clear_if_equal(&mut self.latitude, &existing.latitude);
+        Self::clear_if_equal(&mut self.longitude, &existing.longitude);
+        Self::clear_if_equal(&mut self.asn, &existing.asn);
+        Self::clear_if_equal(&mut self.as_org, &existing.as_org);
+    }
+
+    fn fmt(&self, updates: &mut Vec<String>) {
+        Self::push_value(updates, "continent", &self.continent);
+        Self::push_value(updates, "continent_code", &self.continent_code);
+        Self::push_value(updates, "country", &self.country);
+        Self::push_value(updates, "country_code", &self.country_code);
+        Self::push_value(updates, "city", &self.city);
+        Self::push_value(updates, "latitude", &self.latitude);
+        Self::push_value(updates, "longitude", &self.longitude);
+        Self::push_value(updates, "asn", &self.asn);
+        Self::push_value(updates, "as_org", &self.as_org);
+    }
+}