Release v0.13.0

Bugfixes

.dockerignore

@@ -1,6 +1,9 @@
 target
 Dockerfile
 .dockerignore
+docker
+docs
+ci
 .git
 .gitignore
 *.sw[op]

.travis.yml

@@ -0,0 +1,58 @@
+language: rust
+
+# upload takes too long on windows and gets killed due to inactivity
+#cache: cargo
+
+matrix:
+  include:
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=test
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=common
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=boxxy
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=docker
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=docker-registry
+  - os: osx
+    rust: stable
+    env:
+    - BUILD_MODE=test
+  - os: osx
+    rust: stable
+    env:
+    - BUILD_MODE=common
+  #- os: windows
+  #  rust: stable
+  #  env:
+  #  - BUILD_MODE="windows test"
+  - os: windows
+    rust: stable
+    env:
+    - BUILD_MODE="windows common"
+
+before_install:
+- ci/setup.sh "$TRAVIS_OS_NAME"
+script:
+- df -h
+- ci/run.sh $BUILD_MODE
+- df -h
+
+notifications:
+  irc:
+    channels:
+    - "ircs://irc.hackint.org:6697/#sn0int"
+    #on_success: change # default: always
+    #on_failure: always # default: always
+    use_notice: true

CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# How to contribute
+
+To contribute to sn0int, clone the repository and make sure both the build and
+tests pass for you:
+
+    git clone https://github.com/kpcyrd/sn0int.git
+    cd sn0int
+    # build the project
+    cargo build
+    # run regular tests
+    cargo test
+    # run tests depending on the network
+    # these might fail if a service is down
+    cargo test -- --ignored
+
+The project is loosely structured into a few folders:
+
+- `src/models/` - database models
+- `src/runtime/` - the stdlib that's exposed to lua
+- `src/engine/` - code related to lua
+- `src/sandbox/` - code related to sandboxing
+- `src/cmd/` - cli commands
+- `src/` - misc modules
+
+After you're done, make sure the build completes without any warnings and both
+tests pass successfully:
+
+    cargo test
+    cargo test -- --ignored
+
+If you want to introduce a new feature feel free to open an issue first to make
+sure your feature is a good fit for the project before implementing it.

Cargo.toml

@@ -1,50 +1,107 @@
 [package]
 name = "sn0int"
-version = "0.1.0"
-description = "OSINT framework and package manager"
+version = "0.13.0"
+description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
+edition = "2018"
+
+[badges]
+travis-ci = { repository = "kpcyrd/sn0int" }
 
 [workspace]
 members = ["sn0int-registry/sn0int-common",
            "sn0int-registry"]
 
+[features]
+sqlite-bundled = ["libsqlite3-sys/bundled"]
+
 [dependencies]
-sn0int-common = { version="0.1.0", path="sn0int-registry/sn0int-common" }
-rustyline = "2"
+sn0int-common = { version="0.8.0", path="sn0int-registry/sn0int-common" }
+rustyline = "5.0"
 log = "0.4"
-env_logger = "0.5"
+env_logger = "0.6"
 hlua-badtouch = "0.4"
 structopt = "0.2"
 failure = "0.1"
-rand = "0.5"
+rand = "0.7"
 colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
 publicsuffix = { version="1.5", default-features=false }
-diesel = { version = "1.0.0", features = ["sqlite"] }
+diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-dirs = "1.0"
-url = "1.7"
-nix = "0.11"
-caps = "0.3"
-syscallz = "0.7"
-chrootable-https = "0.3.3"
-trust-dns-proto = "0.5.0-alpha.1"
-base64 = "0.9"
+libsqlite3-sys = "0.12.0"
+chrono = { version = "0.4", features = ["serde"] }
+dirs = "2.0"
+url = "2.0"
+percent-encoding = "2.1"
+#chrootable-https = { path = "../chrootable-https" }
+chrootable-https = "0.11"
+rustls = { version="0.16", features=["dangerous_configuration"] }
+webpki = "0.21"
+webpki-roots = "0.17"
+pem = "0.6.0"
+base64 = "0.10"
+data-encoding = "2.1.2"
 kuchiki = "0.7.2"
-serde_urlencoded = "0.5"
+serde_urlencoded = "0.6"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-crossbeam-channel = "0.2"
+crossbeam-channel = "0.3"
 ctrlc = "3.1"
-opener = "0.3.0"
-separator = "0.3.1"
+opener = "0.4"
+separator = "0.4"
 maplit = "1.0.1"
+sloppy-rfc4880 = "0.1.5"
+regex = "1.0"
+toml = "0.5"
+maxminddb = "0.13"
+tar = "0.4.17"
+libflate = "0.1.14"
+threadpool = "1.7"
+x509-parser = "0.6.0"
+der-parser = "3.0"
+atty = "0.2"
+bufstream = "0.1.4"
+tokio = "0.1.14"
+semver = "0.9"
+bytes = "0.4"
+xml-rs = "0.8"
+bytesize = "1.0"
+ipnetwork = "0.14"
+strum = "0.15"
+strum_macros = "0.15"
+
+digest = "0.8.0"
+bs58 = "0.2.2"
+blake2 = "0.8.0"
+md-5 = "0.8.0"
+sha-1 = "0.8.1"
+sha2 = "0.8.0"
+sha3 = "0.8.0"
+hmac = "0.7"
+
+image = "0.22"
+kamadak-exif = "0.3.1"
+walkdir = "2.2"
+nude = "0.2"
+
+[target.'cfg(target_os="linux")'.dependencies]
+caps = "0.3"
+#syscallz = { path="../syscallz-rs" }
+syscallz = "0.11"
+nix = "0.15"
+
+[target.'cfg(target_os="openbsd")'.dependencies]
+pledge = "0.3.1"
+unveil = "0.2.0"
 
 [dev-dependencies]
-boxxy = "0.7"
+#boxxy = { path = "../boxxy-rs" }
+boxxy = "0.11"
+tempfile = "3.0"

Dockerfile

@@ -1,13 +1,14 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

ISSUE_TEMPLATE.md

@@ -0,0 +1,29 @@
+<!--
+Hello!
+
+If you want to report a bug we added some common questions below that help us analyse your issue faster.
+
+All of these are optional so feel free to remove anything that doesn't apply.
+-->
+
+please describe your issue here
+
+---
+
+## Versions
+
+- **rustc --version:**
+- **cargo --version:**
+- **sn0int --version:**
+- **uname -a:**
+
+## Environment
+
+- **Operating System/Distro:**
+- **Installed from (source/apt/pacman/brew/docker):**
+
+<!--
+Thank you!
+
+We'll try to respond as quickly as possible.
+-->

Makefile

@@ -14,3 +14,15 @@ test:
 	(cd sn0int-registry; cargo test)
 	cargo test
 	cargo test -- --ignored
+
+update:
+	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt
+	get-iab -v -u http://standards-oui.ieee.org/iab/iab.txt -f data/ieee-iab.txt
+	rm -f data/ieee-*.txt.bak
+
+docs:
+	$(MAKE) -C docs html
+	contrib/html-toc2md.pl README.md docs/_build/html/index.html > README2.md
+	mv README2.md README.md
+
+.PHONY: check force-check test update docs

README.md

@@ -1,6 +1,242 @@
-# sn0int
+# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
-work in progress
+[travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
+[travis]:       https://travis-ci.org/kpcyrd/sn0int
+[crates-img]:   https://img.shields.io/crates/v/sn0int.svg
+[crates]:       https://crates.io/crates/sn0int
+[docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
+[docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
+[irc-img]:      https://img.shields.io/badge/hackint-%23sn0int-blue.svg
+[irc]:          https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int
+[twitter-img]:  https://img.shields.io/badge/twitter-@sn0int-blue.svg
+[twitter]:      https://twitter.com/sn0int
+[mastodon-img]: https://img.shields.io/badge/mastodon-chaos.social-blue.svg
+[mastodon]:     https://chaos.social/@sn0int
+[registry-img]: https://img.shields.io/website/https/sn0int.com.svg?label=registry
+[registry]:     https://sn0int.com/
+
+sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
+package manager. It was built for IT security professionals and bug hunters to
+gather intelligence about a given target or about yourself. sn0int is
+enumerating attack surface by semi-automatically processing public information
+and mapping the results in a unified format for followup investigations.
+
+[ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
+
+Among other things, sn0int is currently able to:
+
+- Harvest subdomains from certificate transparency logs and passive dns
+- Enrich ip addresses with asn and geoip info
+- Harvest emails from pgp keyservers and whois
+- Discover compromised logins in breaches
+- Find somebody's profiles across the internet
+- Enumerate local networks with unique techniques like passive arp
+- Gather information about phonenumbers
+- Attempt to bypass cloudflare with shodan
+- Harvest data and images from instagram profiles
+- Scan images for nudity
+
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource. None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
+
+For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
+
+[![asciicast](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB.svg)](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB)
+
+## Installation
+
+Archlinux
+
+    pacman -S sn0int
+
+Mac OSX
+
+    brew install sn0int
+
+For everything else please have a look at the [detailed list][1].
+
+[1]: https://sn0int.readthedocs.io/en/latest/install.html
+
+## Getting started
+
+- [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
+  - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
+  - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
+  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
+  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
+  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
+  - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
+  - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
+- [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
+  - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
+    - [Archlinux](https://sn0int.readthedocs.io/en/latest/build.html#archlinux)
+    - [Mac OSX](https://sn0int.readthedocs.io/en/latest/build.html#mac-osx)
+    - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/build.html#debian-ubuntu-kali)
+    - [Alpine](https://sn0int.readthedocs.io/en/latest/build.html#alpine)
+    - [OpenBSD](https://sn0int.readthedocs.io/en/latest/build.html#openbsd)
+    - [Gentoo](https://sn0int.readthedocs.io/en/latest/build.html#gentoo)
+    - [Windows](https://sn0int.readthedocs.io/en/latest/build.html#windows)
+  - [Building](https://sn0int.readthedocs.io/en/latest/build.html#building)
+- [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
+  - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
+  - [Adding something to scope](https://sn0int.readthedocs.io/en/latest/usage.html#adding-something-to-scope)
+  - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
+  - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
+  - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
+  - [Autonoscope](https://sn0int.readthedocs.io/en/latest/usage.html#autonoscope)
+    - [Domains](https://sn0int.readthedocs.io/en/latest/usage.html#domains)
+    - [IPs](https://sn0int.readthedocs.io/en/latest/usage.html#ips)
+    - [URLs](https://sn0int.readthedocs.io/en/latest/usage.html#urls)
+- [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
+  - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
+  - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
+- [Database](https://sn0int.readthedocs.io/en/latest/database.html)
+  - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
+  - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
+- [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/structs.html#domains)
+  - [Subdomains](https://sn0int.readthedocs.io/en/latest/structs.html#subdomains)
+  - [IpAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#ipaddrs)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/structs.html#urls)
+  - [Emails](https://sn0int.readthedocs.io/en/latest/structs.html#emails)
+  - [Phonenumbers](https://sn0int.readthedocs.io/en/latest/structs.html#phonenumbers)
+  - [Devices](https://sn0int.readthedocs.io/en/latest/structs.html#devices)
+  - [Networks](https://sn0int.readthedocs.io/en/latest/structs.html#networks)
+  - [Accounts](https://sn0int.readthedocs.io/en/latest/structs.html#accounts)
+  - [Breaches](https://sn0int.readthedocs.io/en/latest/structs.html#breaches)
+  - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
+  - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
+  - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
+  - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
+    - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
+    - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
+    - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
+  - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
+  - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
+  - [Using access keys as source argument](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-as-source-argument)
+- [Configuration](https://sn0int.readthedocs.io/en/latest/config.html)
+  - [\[core\]](https://sn0int.readthedocs.io/en/latest/config.html#core)
+  - [\[namespaces\]](https://sn0int.readthedocs.io/en/latest/config.html#namespaces)
+  - [\[network\]](https://sn0int.readthedocs.io/en/latest/config.html#network)
+- [Sandbox](https://sn0int.readthedocs.io/en/latest/sandbox.html)
+  - [Linux](https://sn0int.readthedocs.io/en/latest/sandbox.html#linux)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/sandbox.html#openbsd)
+  - [IPC Protocol](https://sn0int.readthedocs.io/en/latest/sandbox.html#ipc-protocol)
+  - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
+  - [Diagnosing a sandbox failure](https://sn0int.readthedocs.io/en/latest/sandbox.html#diagnosing-a-sandbox-failure)
+- [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
+  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
+  - [base64_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-decode)
+  - [base64_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-encode)
+  - [base64_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-decode)
+  - [base64_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-encode)
+  - [base32_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-decode)
+  - [base32_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-encode)
+  - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
+  - [create_blob](https://sn0int.readthedocs.io/en/latest/reference.html#create-blob)
+  - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
+  - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
+  - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
+  - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
+  - [error](https://sn0int.readthedocs.io/en/latest/reference.html#error)
+  - [geoip_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#geoip-lookup)
+  - [hex](https://sn0int.readthedocs.io/en/latest/reference.html#hex)
+  - [hmac_md5](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-md5)
+  - [hmac_sha1](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha1)
+  - [hmac_sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-256)
+  - [hmac_sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-512)
+  - [hmac_sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-256)
+  - [hmac_sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-512)
+  - [html_select](https://sn0int.readthedocs.io/en/latest/reference.html#html-select)
+  - [html_select_list](https://sn0int.readthedocs.io/en/latest/reference.html#html-select-list)
+  - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
+  - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
+  - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [http_fetch](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch)
+  - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
+  - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
+  - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
+  - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
+  - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
+  - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
+  - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
+  - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
+  - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
+  - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
+  - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
+  - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
+  - [set_err](https://sn0int.readthedocs.io/en/latest/reference.html#set-err)
+  - [sha1](https://sn0int.readthedocs.io/en/latest/reference.html#sha1)
+  - [sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-256)
+  - [sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-512)
+  - [sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-256)
+  - [sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-512)
+  - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
+  - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
+  - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
+  - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
+  - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
+  - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
+  - [sock_recvline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline)
+  - [sock_recvall](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvall)
+  - [sock_recvline_contains](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline-contains)
+  - [sock_recvline_regex](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline-regex)
+  - [sock_recvn](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvn)
+  - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
+  - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
+  - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
+  - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
+  - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
+  - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
+  - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
+  - [url_escape](https://sn0int.readthedocs.io/en/latest/reference.html#url-escape)
+  - [url_join](https://sn0int.readthedocs.io/en/latest/reference.html#url-join)
+  - [url_parse](https://sn0int.readthedocs.io/en/latest/reference.html#url-parse)
+  - [url_unescape](https://sn0int.readthedocs.io/en/latest/reference.html#url-unescape)
+  - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
+  - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
+  - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
+  - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
+  - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
+  - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)
+
+## Rationale
+
+This tool was written for companies to help them understand their attack
+surface from a blackbox point of view. It's often difficult to understand that
+something is easier to discover than some people assume, putting them at risk
+of false security.
+
+It's also designed to be useful for red team assessments and bug bounties,
+which also help companies to identify weaknesses that could result in a
+compromise.
+
+Some functionality was written to do the same thing for individuals to raise
+awareness about personal attack surface, privacy and how much data is publicly
+available. These issues are often out of scope in bug bounties and sometimes by
+design. We believe that blaming the user is the wrong approach and these issues
+should be addressed at the root cause by the people designing those systems.
 
 ## License
 

ci/bench.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+set -eu
+X=$(mktemp -d)
+cd "$X"
+
+mkdir -p "$X/.cache"
+cp -r "$HOME/.cache/sn0int" "$X/.cache/"
+
+#export CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
+#export RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}"
+export HOME="$X"
+
+cat > 1k.lua <<EOF
+-- Description: Insert 1k random subdomains
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run(arg)
+    for i=1,1000 do
+        x = 'foo' .. i .. '.example.com'
+
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=x,
+        })
+    end
+end
+EOF
+
+echo '[*] Setting up workspace'
+echo 'add domain example.com' | "$@" > /dev/null
+echo '[*] Running 1k inserts'
+time "$@" run -f ./1k.lua

ci/boxxy_stage1.txt

@@ -0,0 +1,5 @@
+stage1
+ls
+echo checkpoint1
+id
+echo sandbox fail

ci/integration.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+import subprocess
+from subprocess import DEVNULL, PIPE
+import tempfile
+import json
+import sys
+
+
+def _sn0int(tempdir, binary, args, piped_stdout=False):
+    return subprocess.Popen(
+        ['/usr/bin/env', 'HOME='+tempdir, binary] + args,
+        stdin=PIPE,
+        stdout=PIPE if piped_stdout else None,
+    )
+
+
+def sn0int(tempdir, binary, cmds):
+    p = _sn0int(tempdir, binary, [])
+    for cmd in cmds:
+        p.stdin.write((cmd + '\n').encode('utf-8'))
+    p.communicate()
+    if p.returncode != 0:
+        raise Exception('process failed')
+
+
+def sn0int_select(tempdir, binary, query):
+    p = _sn0int(tempdir, binary, ['select', '--json'] + query, piped_stdout=True)
+    stdout, _ = p.communicate()
+    lines = filter(None, stdout.decode('utf-8').split('\n'))
+    return [json.loads(x) for x in lines]
+
+
+def main(tempdir, binary):
+    print('[*] setting up workspace')
+    sn0int(tempdir, binary, [])
+
+    print('[*] adding domain')
+    sn0int(tempdir, binary, [
+        'add domain',
+        'example.com',
+        'select domains',
+    ])
+
+    print('[*] testing db for domain')
+    domains = sn0int_select(tempdir, binary, ['domains'])
+    assert domains == [{'id': 1, 'value': 'example.com', 'unscoped': False}]
+
+    print('[*] installing modules')
+    sn0int(tempdir, binary, [
+        'mod install kpcyrd/ctlogs',
+        'mod install kpcyrd/dns-resolve',
+        'mod install kpcyrd/url-scan',
+        'mod install kpcyrd/geoip',
+    ])
+
+    print('[*] running ctlogs')
+    sn0int(tempdir, binary, [
+        'use ctlogs',
+        'run',
+        'select subdomains',
+    ])
+
+    print('[*] testing db for subdomains')
+    subdomains = sn0int_select(tempdir, binary, ['subdomains'])
+    assert {x['value'] for x in subdomains} == {
+        'www.example.com',
+        'm.example.com',
+        'dev.example.com',
+        'products.example.com',
+        'support.example.com',
+    }
+
+    print('[*] running dns-resolve')
+    sn0int(tempdir, binary, [
+        'use dns-resolve',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs')
+    ipaddrs = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert len(ipaddrs) >= 1
+
+    print('[*] running url-scan')
+    sn0int(tempdir, binary, [
+        'use url-scan',
+        'run',
+        'select urls',
+    ])
+
+    print('[*] testing db for urls')
+    urls = sn0int_select(tempdir, binary, ['urls'])
+    assert {(x['value'], x['status']) for x in urls} == {
+        ('http://www.example.com/', 200),
+        ('https://www.example.com/', 200),
+    }
+
+    print('[*] running geoip')
+    sn0int(tempdir, binary, [
+        'use geoip',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs again')
+    ipaddrs2 = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert ipaddrs != ipaddrs2
+
+    print('')
+    print('\t###########')
+    print('\t# SUCCESS #')
+    print('\t###########')
+    print('')
+
+
+if __name__ == '__main__':
+    try:
+        binary = sys.argv[1]
+    except IndexError:
+        print('Usage: %s target/release/sn0int' % sys.argv[0])
+    else:
+        with tempfile.TemporaryDirectory(prefix='sn0int-') as tempdir:
+            main(tempdir, binary)

ci/reprotest.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -xue
+
+# tested with rustc 1.30.0 and cargo 1.30.0
+
+# by default, the build folder is located in /tmp, which is a tmpfs. The target/ folder
+# can become quite large, causing the build to fail if we don't have enough RAM.
+export TMPDIR="$HOME/tmp/repro-test"
+mkdir -p "$TMPDIR"
+
+# fileordering: https://github.com/diesel-rs/diesel/pull/1902
+# build_path: https://github.com/briansmith/ring/issues/715
+
+reprotest -vv --vary=-time,-domain_host,-fileordering,-build_path --source-pattern 'Cargo.* src/ sn0int-registry/ migrations/' '
+    CARGO_HOME="$PWD/.cargo" RUSTUP_HOME='"$HOME/.rustup"' \
+        RUSTFLAGS="--remap-path-prefix=$HOME=/remap-home --remap-path-prefix=$PWD=/remap-pwd" \
+        cargo build --release --verbose --locked' \
+    target/release/sn0int

ci/run.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+set -exu
+case "$1" in
+    build)
+        cargo build --verbose
+        cargo build --verbose --examples
+        ;;
+    test)
+        ci/run.sh build
+        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
+             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
+        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
+        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
+        cargo test --verbose
+        cargo test --verbose -- --ignored
+        ;;
+    common)
+        cd sn0int-registry/sn0int-common
+        cargo test --verbose
+        ;;
+    windows)
+        cargo build --verbose --features=sqlite-bundled
+        cargo build --verbose --examples --features=sqlite-bundled
+        ;;
+    boxxy)
+        cargo build --verbose --examples
+        if cat ci/boxxy_stage1.txt | RUST_LOG=boxxy cargo run --example boxxy; then
+            echo "SANDOX ERROR: should've crashed"
+            exit 1
+        fi
+        ;;
+    docker)
+        docker build -t sn0int .
+        docker images
+        docker run --rm sn0int --help
+        ;;
+    docker-registry)
+        docker build -t sn0int-registry sn0int-registry/
+        docker images
+        ;;
+esac

ci/setup.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -exu
+case "$1" in
+    linux)
+        sudo apt update
+        sudo apt install libsqlite3-dev libseccomp-dev
+        ;;
+esac

contrib/docker/Dockerfile.alpine

@@ -0,0 +1,15 @@
+FROM alpine:edge
+RUN apk add --no-cache sqlite-dev libseccomp-dev
+RUN apk add --no-cache --virtual .build-rust rust cargo
+WORKDIR /usr/src/sn0int
+COPY . .
+RUN cargo build --release --verbose
+RUN strip target/release/sn0int
+
+FROM alpine:edge
+RUN apk add --no-cache libgcc sqlite-libs libseccomp
+COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+VOLUME ["/data", "/cache"]
+ENV XDG_DATA_HOME=/data \
+    XDG_CACHE_HOME=/cache
+ENTRYPOINT ["sn0int"]

contrib/docker/Dockerfile.debian

@@ -0,0 +1,16 @@
+FROM rust
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /usr/src/sn0int
+COPY . .
+RUN cargo build --release --verbose
+RUN strip target/release/sn0int
+
+FROM debian
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+VOLUME ["/data", "/cache"]
+ENV XDG_DATA_HOME=/data \
+    XDG_CACHE_HOME=/cache
+ENTRYPOINT ["sn0int"]

contrib/html-toc2md.pl

@@ -0,0 +1,39 @@
+#!/usr/bin/env perl
+use strict; use warnings;
+
+my ($readme, $toc) = @ARGV;
+defined $readme or die 'missing readme path';
+defined $toc or die 'missing toc path';
+
+open(my $r, "<$readme") or die 'failed to open readme';
+open(my $t, "<$toc") or die 'failed to open toc';
+
+my $re = qr/^\s*- \[.+\]\(https:\/\/sn0int.readthedocs.io\/en\/.+\)$/;
+
+# pass through start of readme
+while (<$r>) {
+    last if ($_ =~ $re);
+    print $_;
+}
+
+# skip toc
+while (<$r>) {
+    last unless ($_ =~ $re);
+}
+
+# generate new toc
+while (my $line = <$t>) {
+    if ($line =~ /toctree-l(\d).*href="([^"]+)">(.+)<\/a/) {
+        my $space = int($1)-1;
+        my $section = $2;
+        my $label = $3;
+        $label =~ s/([\[\]])/\\$1/g;
+        print " " x ($space*2), "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
+    }
+}
+print;
+
+# pass through end of readme
+while (<$r>) {
+    print $_;
+}

docs/.gitignore

@@ -0,0 +1 @@
+/_build/

docs/Makefile

@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/build.rst

@@ -0,0 +1,80 @@
+Build from source
+=================
+
+It's generally recommended to `install a package <install.html>`_ if available.
+This section is about building the binary from git.
+
+Install dependencies
+--------------------
+
+You need a recent rust compiler. It's usually recommended to install a rust
+compiler with `rustup <https://rustup.rs/>`_, but if you're system ships the
+most recent compiler in a package that works too. Note that some systems aren't
+fully supported by rustup (like OpenBSD and alpine) and you need to install
+rust from a package in that case.
+
+Archlinux
+~~~~~~~~~
+
+.. code-block:: bash
+
+    $ pacman -S geoip2-database libseccomp publicsuffix-list sqlite
+
+Mac OSX
+~~~~~~~
+
+None.
+
+Debian/Ubuntu/Kali
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+
+.. warning::
+   On a debian based system make sure you've installed rust with rustup.
+
+Alpine
+~~~~~~
+
+.. code-block:: bash
+
+    $ apk add sqlite-dev libseccomp-dev
+
+OpenBSD
+~~~~~~~
+
+.. code-block:: bash
+
+    $ pkg_add sqlite3 geolite2-city geolite2-asn
+
+Gentoo
+~~~~~~
+
+.. code-block:: bash
+
+    emerge --ask sys-libs/libseccomp dev-db/sqlite
+
+Windows
+~~~~~~~
+
+You don't need to install any dependencies on windows, but you need to use a
+different build command in the next section.
+
+Building
+--------
+
+After all dependencies have been installed, simply build the binary:
+
+.. code-block:: bash
+
+    $ cargo build --release
+
+Note that you need a different command on windows:
+
+.. code-block:: bash
+
+    $ cargo build --release --features=sqlite-bundled
+
+After the build finished the binary is located at ``target/release/sn0int``.

docs/conf.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'sn0int'
+copyright = '2018, kpcyrd'
+author = 'kpcyrd'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = ''
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'sn0intdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'sn0int.tex', 'sn0int Documentation',
+     'kpcyrd', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('man', 'sn0int', 'Semi-automatic OSINT framework and package manager',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'sn0int', 'sn0int Documentation',
+     author, 'sn0int', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']

docs/config.rst

@@ -0,0 +1,54 @@
+Configuration
+=============
+
+This section documents the config file. By default this file does not exist and
+a default configuration is used instead.
+
+Linux/BSD
+    ``~/.config/sn0int.toml``
+
+OSX
+    ``~/Library/Preferences/sn0int.toml``
+
+Windows
+    ``%APPDATA%/sn0int.toml``
+
+[core]
+------
+
+``registry``
+    Configure the registry you want to use. Defaults to ``https://sn0int.com``.
+``no-autoupdate``
+    sn0int is going to check if your modules are outdated during startout once
+    a week. Set this option to ``true`` to disable this.
+
+[namespaces]
+------------------
+
+By default sn0int modules are assumed to be installed from the registry. You
+may want to keep a local directory with private modules, especially during
+development. You can configure a folder that contains modules that aren't
+managed by sn0int by adding a namespace section to the config file::
+
+    [namespaces]
+    foo = "/opt/sn0int/foo"
+    bar = "~/repos/a/b/c/sn0int-modules"
+
+This is going to load modules from these two folders and register them in the
+``foo`` and ``bar`` namespace.
+
+Note that sn0int is also going to assume that symlinks in
+``~/.local/share/sn0int/modules`` and folders containing a ``.git`` folder are
+externally managed.
+
+[network]
+---------
+
+To enable a proxy, add the following to your config file::
+
+    [network]
+    proxy = "127.0.0.1:9050"
+
+This forces everything through tor (or any other socks5 proxy) and restricts
+all other functions that depend on the network. For example the ``dns``
+function is fully disabled if a proxy is configured.

docs/database.rst

@@ -0,0 +1,84 @@
+Database
+========
+
+There are a few things you need to understand how the database works to use it
+efficiently.
+
+The database that is backing sn0int is sqlite, but the api that is exposed to
+the user and scripts is an nosql-ish object store. The query language that is
+exposed to the user is still very similar to sql, except that it lacks a column
+statement::
+
+    select subdomains where value like %.example.com
+    ^      ^          ^           ^    ^
+    |      |          |           |    this value is going to be quoted automatically
+    |      |          |           |
+    |      |          |           this triggers automatic quoting
+    |      |          |
+    |      |          apply a filter, this translates to sql quite literally
+    |      |
+    |      the entity we want to select is a subdomain.
+    |      this affects the table and the deserializer
+    |
+    select entities
+
+This is how almost all user facing functions work that operate on the database.
+The functions that are available for scripting are a bit more object based and
+described below.
+
+db_add
+------
+
+This operation is somewhat straight forward. It adds an entity to the
+database:
+
+.. code-block:: lua
+
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+
+If this entity conflicts with an entity that already exists, an upsert is
+triggered and an db_update is performed instead.
+
+.. note::
+   This function may return ``nil`` if the entity already exists, but has been
+   removed from scope with ``noscope``. Everytime you use ``db_add`` you need
+   to make sure that the ID that has been returned is not ``nil``.
+
+db_update
+---------
+
+Update some mutable fields of an entity:
+
+.. code-block:: lua
+
+    db_update('ipaddr', arg, {
+        asn=lookup['asn'],
+        as_org=lookup['as_org'],
+    })
+
+The first parameter is usually the same arg that your script was called with.
+Usually you can use db_add instead of db_update due to the upsert feature, but
+db_update is still slightly faster.
+
+.. note::
+   Some fields are immutable and can not be updated.
+
+db_select
+---------
+
+This function is used to check if something is in scope. If the entity has been
+added to the database and has not been removed from scope, this function
+returns that entities id. This is somewhat similar to ``db_add``, except that
+``db_select`` never adds anything to the database.
+
+.. code-block:: lua
+
+    domain_id = db_select('domain', 'example.com')
+    if domain_id ~= nil then
+        -- do something
+    end
+
+This function only accepts a string instead of a lua table. This string is used
+to filter on the ``value`` column.

docs/index.rst

@@ -0,0 +1,47 @@
+sn0int
+======
+
+sn0int is a semi-automatic OSINT framework and package manager. It was built
+for IT security professionals and bug hunters to gather intelligence about a
+given target or about yourself. sn0int is enumerating attack surface by
+semi-automatically processing public information and mapping the results in a
+unified format for followup investigations.
+
+Among other things, sn0int is currently able to:
+
+- Harvest subdomains from certificate transparency logs
+- Harvest subdomains from various passive dns logs
+- Sift through subdomain results for publicly accessible websites
+- Harvest emails from pgp keyservers
+- Enrich ip addresses with ASN and geoip info
+- Harvest subdomains from the wayback machine
+- Gather information about phonenumbers
+- Bruteforce interesting urls
+
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource.  None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
+
+Join us on IRC: `irc.hackint.org:6697/#sn0int <https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int>`_
+
+Getting Started
+---------------
+
+.. toctree::
+   :maxdepth: 3
+   :glob:
+
+   install
+   build
+   usage
+   scripting
+   database
+   structs
+   keyring
+   config
+   sandbox
+   reference

docs/install.rst

@@ -0,0 +1,85 @@
+Installation
+============
+
+If available, please prefer the package shipped by operating system. If your
+operating system has a package but you're running on older version, please use
+the `build from source <build.html>`_ instructions instead.
+
+Archlinux
+---------
+
+.. code-block:: bash
+
+    $ pacman -S sn0int
+
+Mac OSX
+-------
+
+.. code-block:: bash
+
+    $ brew install sn0int
+
+Debian/Ubuntu/Kali
+------------------
+
+Note that debian `doesn't ship the geoip2-database
+<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
+download them automatically during the first run.
+
+Using rust+cargo from the repos might work for you, but we only officially
+support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
+at the docker image as an alternative.
+
+.. code-block:: bash
+
+    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f --path .
+
+Docker
+------
+
+.. code-block:: bash
+
+    $ docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
+
+Alpine
+------
+
+On alpine edge, with enabled testing repositories:
+
+.. code-block:: bash
+
+    $ apk add sn0int
+
+OpenBSD
+-------
+
+On -current:
+
+.. code-block:: bash
+
+    $ pkg_add sn0int
+
+Gentoo
+------
+
+.. code-block:: bash
+
+    layman -f -o https://raw.githubusercontent.com/kpcyrd/overlay/master/overlay.xml -a kpcyrd-overlay
+    emerge --ask net-analyzer/sn0int
+
+Windows
+-------
+
+This is not recommended and only passively maintained. Please prefer linux in a
+virtual machine if needed.
+
+Make sure rust is installed and setup.
+
+.. code-block:: bash
+
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f --path . --features=sqlite-bundled

docs/keyring.rst

@@ -0,0 +1,73 @@
+Keyring
+=======
+
+A common problem is that you need either an api key or a username/password
+combination. Instead of hardcoding it in the script you should request them
+from the keyring. In order to do this you need to request permissions to those
+credentials.
+
+Managing the keyring
+--------------------
+
+The keyring is a simple namespaced key-value store::
+
+    [sn0int][default] > keyring add aws:AKIAIOSFODNN7EXAMPLE
+    Secretkey: keep-this-secret
+    [sn0int][default] > keyring list
+    aws:AKIAIOSFODNN7EXAMPLE
+    [sn0int][default] >
+    [sn0int][default] > keyring list aws
+    aws:AKIAIOSFODNN7EXAMPLE
+    [sn0int][default] > keyring list instagram
+    [sn0int][default] >
+    [sn0int][default] > keyring get aws:AKIAIOSFODNN7EXAMPLE
+    Namespace:    "aws"
+    Access Key:   "AKIAIOSFODNN7EXAMPLE"
+    Secret:       "keep-this-secret"
+    [sn0int][default] >
+
+If the service uses a username-password combination, set the username as the
+access key and the password as the secret.
+
+If the service uses only a secret key for the api, set the secret key as the
+access key and leave the secret blank.
+
+A script doesn't automatically get access to requested keyring namespaces.
+Instead the user is asked to confirm those requests to limit abusive scripts.
+
+Using access keys in scripts
+----------------------------
+
+We can request all keys of a certain namespace in our script metadata. This is
+going to prompt the user to grant the script access. This can be done for
+multiple namespaces in the same script:
+
+.. code-block:: lua
+
+    -- Keyring-Access: aws
+    -- Keyring-Access: asdf
+
+If the user granted us access to those keys we can read them with ``keyring``:
+
+.. code-block:: lua
+
+    creds = keyring('aws')
+    debug(creds[1]['access_key'])
+    debug(creds[1]['secret_key'])
+
+This returns a list of all keys in that namespace. Any empty list is returned
+if the user doesn't have any keys in that namespace.
+
+Using access keys as source argument
+------------------------------------
+
+We can also use the access keys as source argument. This is useful if each
+account has access to different things and we want to read through all of them.
+
+Since access key permissions are granted per namespace we need to specify which
+credentials we want to use.
+
+.. code-block:: lua
+
+    -- Keyring-Access: aws
+    -- Source: keyring:aws

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/man.rst

@@ -0,0 +1,14 @@
+:orphan:
+
+sn0int
+======
+
+todo
+
+.. toctree::
+    :maxdepth: 3
+    :glob:
+
+    usage
+    config
+    reference

docs/sandbox.rst

@@ -0,0 +1,147 @@
+Sandbox
+=======
+
+Scripts are generally considered to be untrusted and executed exclusively in a
+child process. It's important to note that there's a basic sandbox that's
+active on every operating system, and there's a second line of defense on
+supported operating systems.
+
+The first line of defense is the restrictive stdlib. It's assumed that and
+attacker gains full control over the lua code and is able to call any function
+with arbitrary arguments. The stdlib only provides functions that are
+considered safe, so for example it's not possible to start a process or open a
+file.
+
+The second line of defense is supposed to make sure the system isn't
+compromised even if the first layer is fully broken and an attacker gains full
+control over the child process.
+
+Right now this is only supported on linux and openbsd.
+
+Linux
+-----
+
+On linux we use seccomp to filter all syscalls that we don't need. We also use
+chroot to disable filesystem access. It's recommended to install the sn0int
+binary with ``cap_sys_chroot`` to make sure unprivileged users can use chroot.
+The chroot location is hard coded and all capabilities are removed after the
+chroot is done or if no chroot is going to happen.
+
+OpenBSD
+-------
+
+On openbsd we're using ``pledge`` to restrict syscalls and ``unveil`` to
+restrict filesystem access.
+
+IPC Protocol
+------------
+
+The parent process and the child process communicate using an IPC protocol that
+is line-based json.
+
+For a simple hello world the parent process is only going to send a single line
+to the child process. This line contains:
+
+- The function argument
+- The dns config
+- Keys that the module has been given access to
+- The module metadata and code
+- Options, if any
+- A socks5 proxy, if any
+- The log level
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    -- nothing to do here\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+
+Saving this line in a file called ``start.json`` and sending it to a sandbox
+process should result in the following output::
+
+    $ sn0int sandbox foobar < start.json
+    {"Exit":"Ok"}
+    $
+
+This line tells us that the script terminated successfully.
+
+There are some functions that cause a notification to the parent process. We
+are going to add a call to the ``info()`` function to our module:
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    info('ohai')\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+
+This is going to print an additional event::
+
+    $ sn0int sandbox foobar < start2.json
+    {"Log":{"Info":"\"ohai\""}}
+    {"Exit":"Ok"}
+    $
+
+There are some functions that block the child process until the parent process
+sent a reply. These functions are mostly database related functions, since the
+child doesn't have direct database access. To demonstrate this, we're going to
+write two lines to our file this time, one is the init line and the second one
+is the reply for the database event:
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    x = db_add('domain', {value=\"example.com\"})\n    info(x)\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+    {"Ok":1337}
+
+Results in the following output::
+
+    $ target/release/sn0int sandbox foobar < start3.json
+    {"Database":{"Insert":{"Domain":{"value":"example.com"}}}}
+    {"Log":{"Info":"1337.0"}}
+    {"Exit":"Ok"}
+    $
+
+The first line is a database event and indicates that the child wants to insert
+data. After printing this line the child tries to read a line from stdin, this
+is why we needed to write two lines to our json file this time. In the second
+line the child learns if the insert was successful and which id was assigned to
+that entity.
+
+Limitations
+-----------
+
+There are some limitations that you should be aware:
+
+- Network access is available and network namespaces aren't isolated. This
+  means scripts have access to your local network, the internet and also your
+  localhost loopback interface.
+- If chroot is unavailable an attacker could connect to unix domain sockets.
+
+Diagnosing a sandbox failure
+----------------------------
+
+You might experience a sandbox failure, especially on architectures that are
+less popular. This usually looks like this::
+
+    [sn0int][example][kpcyrd/ctlogs] > run
+    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [+] Finished kpcyrd/ctlogs (1 errors)
+
+A module that never finishes could also mean an IO thread inside the worker got
+killed by the sandbox.
+
+You can try to diagnose this yourself with strace::
+
+    strace -f sn0int run -vv ctlogs 2>&1 | tee strace.log
+
+Open ``strace.log``, look out for syscalls that didn't return by searching for
+``= ?`` and ignore calls to exit and similar. You are looking for something
+like this::
+
+    seccomp(SECCOMP_SET_MODE_FILTER, 0, {len=48, filter=0xdd59094e490}) = 0
+    write(1, "[+] activated!\n", 15[+] activated!
+    )        = 15
+    getresuid( <unfinished ...>)            = ?
+    +++ killed by SIGSYS (core dumped) +++
+
+This would indicate a call to ``getresuid`` which was not allowed by the
+seccomp filter.
+
+If you don't want to diagnose this yourself open a new bug report with as much
+information as possible, specifically which distro, which release and which
+architecture you're using.

docs/scripting.rst

@@ -0,0 +1,243 @@
+Scripting
+=========
+
+Scripting is the core feature in sn0int. It's not strictly required, but if you
+want to write your own modules, this section is for you.
+
+Write your first module
+-----------------------
+
+It's highly recommended to use a VCS for development, so let's start by setting
+that up. We're going to assume you store your repos in ``~/repos`` but you're
+free to change that to something else::
+
+    $ git init ~/repos/sn0int-modules
+    $ cd ~/repos/sn0int-modules
+
+We need to add this folder to the sn0int config file so it's correctly detected
+when starting sn0int. Open the `config file <config.html>`_ in your prefered
+editor. Note that the file does not exist by default and the path is different
+depending on your operating system. On linux you would open the config file
+with::
+
+    $ vim ~/.config/sn0int.toml
+
+Add the following::
+
+    [namespaces]
+    your_github_name = "~/repos/sn0int-modules"
+
+Every module we're adding to ``~/repos/sn0int-modules`` is now going to be
+picked up by sn0int.
+
+Make sure you're still in the right folder and add your first module::
+
+    sn0int new first.lua
+
+This is going to generate some boilerplate for you that every module needs to
+load successfully. Afterwards we can edit it like this:
+
+.. code-block:: lua
+
+    -- Description: ohai wurld
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
+    function run(arg)
+        -- TODO: do something here
+    end
+
+``Description`` (mandatory)
+  This should be a short text that describes what your module is doing.
+
+``Version`` (mandatory)
+  Every module requires a semver_ version. You can just set it to ``0.1.0``
+  during development, but you need to increase it every time you publish your
+  module. If you don't care about that one, just keep increasing ``0.X.0``.
+
+.. _semver: https://semver.org/
+
+``Source`` (mandatory)
+  This is going to specify what kind of entities we're interested in. If we
+  specify ``domains`` our module is going to be called with all domains that
+  are targeted.
+
+  - ``domains``
+  - ``subdomains``
+  - ``ipaddrs``
+  - ``urls``
+  - ``emails``
+
+``License`` (mandatory)
+  This is somewhat special. We require that every module is licensed under an
+  open source license. Pick one of the following licenses.
+
+  - ``MIT`` -  https://opensource.org/licenses/MIT
+  - ``GPL-3.0`` - https://opensource.org/licenses/gpl-license
+  - ``LGPL-3.0`` - https://opensource.org/licenses/lgpl-license
+  - ``BSD-2-Clause`` - https://opensource.org/licenses/BSD-2-Clause
+  - ``BSD-3-Clause`` - https://opensource.org/licenses/BSD-3-Clause
+  - ``WTFPL`` - https://spdx.org/licenses/WTFPL.html
+
+``function run(arg)`` (mandatory)
+  This is where the actual magic of our module happens. Our function is going
+  to be called in a loop for each entity that is targeted by the user.
+
+Let's continue. For the sake of an hello world we're going to take some
+``domains``, check if a ``www`` subdomain exists and if it does, add it to the
+database.
+
+.. code-block:: lua
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+        print(subdomain)
+    end
+
+Combined with the header we wrote previously we can already execute this
+module. Make sure you've added a domain to scope with ``add domain
+example.com``, save your file and run it like this::
+
+    sn0int run -f ./first.lua
+
+We should see some output by our print function.
+
+.. note::
+   ``print`` is useful for development but must be removed before publishing.
+
+Next, we want to actually resolve that name, we're going to use the ``dns``
+function for that. This function takes a name and a query type and returns a
+result. Note that this function might fail, in which case we want to abort our
+function. We do that by checking if the return value of ``last_err()`` is
+truth-y.
+
+.. code-block:: lua
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+
+        records = dns(subdomain, {
+            record='A'
+        })
+        if last_err() then return end
+
+        print(records)
+    end
+
+If you run your module again you're going to see some output, either
+``{"answers":[somedata],"error":null}`` or
+``{"answers":[],"error":"NXDomain"}``. We decide that we add the subdomain to
+our scope and set it to resolvable if ``error`` is ``nil``.
+
+.. code-block:: lua
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+
+        records = dns(subdomain, {
+            record='A'
+        })
+        if last_err() then return end
+
+        if records['error'] == nil then
+            db_add('subdomain', {
+                domain_id=arg['id'],
+                value=subdomain,
+                resolvable=true,
+            })
+        end
+    end
+
+.. hint::
+   See the database section to understand how the database works in detail.
+
+If we execute our module one more time it's going to log that it discovered a
+subdomain, if it doesn't, try adding more domains to scope. Note that this only
+happens the first time. Modules that don't discover anything or don't discover
+anything new exit silently.
+
+After putting everything together, our final module looks like this:
+
+.. code-block:: lua
+
+    -- Description: ohai wurld
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+
+        records = dns(subdomain, {
+            record='A'
+        })
+        if last_err() then return end
+
+        if records['success'] ~= nil then
+            db_add('subdomain', {
+                domain_id=arg['id'],
+                value=subdomain,
+                resolvable=true,
+            })
+        end
+    end
+
+There's still some room for improvement, for example, since we already resolved
+that record, we could also add the ip address to the scope and link it to the
+subdomain we added.
+
+Publish your module
+-------------------
+
+The public registry uses github usernames to namespace the registry. This means
+you need to authenticate to the registry using your github username. This can
+be done using::
+
+    sn0int login
+
+sn0int is going to open a new tab in your browser, if you are already signed
+into your github account you only need to confirm an authorization request. The
+application doesn't need any of your data, so it's only asking you to confirm
+your identity.
+
+Afterwards publish your module with::
+
+    sn0int publish ./first.lua
+
+Reading data from stdin
+-----------------------
+
+Sometimes you need to read data that can't be easily accessed from within the
+sandbox, like output of other programms or file content. In that case you can
+write a module that reads from stdin:
+
+.. code-block:: lua
+
+    -- Description: Read from stdin
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+
+    function run()
+        while true do
+            x = stdin_readline()
+            if x == nil then
+                break
+            end
+            info(x)
+        end
+    end
+
+Write it to a file and run it like this::
+
+    % echo hello | sn0int run --stdin -vvf stdin.lua
+    [*] anonymous/stdin                                   : "hello\n"
+    [+] Finished anonymous/stdin
+    %
+
+This is going to read one line at a time and allows you to process it with
+regular expressions and add data to the database.
+
+.. note::
+   If you get an error like ``Failed to read stdin: "stdin is unavailable"``
+   make sure the ``--stdin`` flag is set.

docs/structs.rst

@@ -0,0 +1,306 @@
+Structs
+=======
+
+This section describes all supported structs in depth. Please refer to this
+section if in doubt about the correct usage of fields to ensure
+interoperability between modules.
+
+Domains
+-------
+
+Represents a registerable domain as defined by the `public suffix list
+<https://publicsuffix.org/>`_. If in doubt check `psl_domain_from_dns_name
+<reference.html#psl-domain-from-dns-name>`_.
+
+``value``
+  The domain name, like ``example.co.uk``.
+
+Subdomains
+----------
+
+A subdomain of a `domain <#domains>`_. The depth is arbitrary, so
+``foo.example.co.uk`` and ``foo.bar.example.co.uk`` are both valid subdomains
+of ``example.co.uk``.
+
+``value``
+  The subdomain, like ``foo.bar.example.co.uk``.
+``domain_id``
+  The numeric id of a domain struct.
+``resolvable``
+  Whether the subdomain can be resolved to a A/AAAA record. nil if unknown.
+
+IpAddrs
+-------
+
+An ip address. Note that most of these fields are geoip related and an
+approximation instead of an actual location.
+
+``value``
+    The ip address.
+``family``
+    The address family of the ip address, either ``4`` or ``6``.
+``continent``
+    The continent associated with this ip address.
+``continent_code``
+    The continent code of the ``continent`` field, eg ``NA``.
+``country``
+    The country associated with this ip address.
+``country_code``
+    The country code of the ``country`` field, eg ``US``.
+``city``
+    The city associated with this ip address.
+``latitude``
+    Latitude associated with this ip address.
+``longitude``
+    Longitude associated with this ip address.
+``asn``
+    The number of the autonomous system this ip belongs to.
+``as_org``
+    The organization of the autonomous system this ip belongs to.
+``description``
+    This field is sn0int internal if we have additional information about this
+    ip address, for example technical identifiers from aws.
+``reverse_dns``
+    The reverse dns name setup for this ip address.
+
+URLs
+----
+
+``subdomain_id``
+    The numeric id of a subdomain struct.
+``value``
+    The url, including a schema, hostname and path.
+``status``
+    The http status code, like ``200``.
+``body``
+    The raw response body. This can be any mime type.
+``online``
+    Whether or not the url gives a http response (even if it's an error).
+``title``
+    The parsed ``<title>`` of the page, if available.
+``redirect``
+    If the server replied with a redirect, this is the url it redirected to.
+
+Emails
+------
+
+``value``
+    The email address.
+``displayname``
+    The display name of a given email address: ``this is the name <foo@example.com>``.
+``valid``
+    Whether that email address is valid or has been disabled.
+
+Phonenumbers
+------------
+
+``value``
+    The phone number in E.164 format (+491234567)
+``name``
+    An alias we can assign to this phone number. This alias is sn0int internal.
+``valid``
+    Whether the number is assigned to a customer.
+``last_online``
+    The last time this number has been online.
+``country``
+    The country this number is associated with.
+``carrier``
+    The name of the carrier this numer is registered with.
+``line``
+    The type of the phone number, can be ``landline``, ``mobile`` or ``voip``.
+``is_ported``
+    Whether this number has been ported to a different carrier.
+``last_ported``
+    The last time this number has been ported.
+``caller_name``
+    The name of the owner of the phone number.
+``caller_type``
+    The type of caller, eg ``business`` or ``consumer``.
+
+Devices
+-------
+
+``value``
+    The devices mac address or another identifier if needed.
+``name``
+    An alias we can assign to this device. This alias is sn0int internal.
+``hostname``
+    The hostname configured on the device.
+``vendor``
+    The hardware vendor of the device. This is usually derived from the mac
+    address.
+``last_seen``
+    The last time we've observed the device somewhere.
+
+Networks
+--------
+
+A wired or wireless network at a specific location that a device could be
+connected to.
+
+``value``
+    The network name. This can be an ssid or any other identifier but should be
+    unique.
+``latitude``
+    Latitude of the networks location.
+``longitude``
+    Longitude of the networks location.
+``description``
+    A human readable description in case the value is a technical identifier.
+
+Accounts
+--------
+
+A users account or profile on a webservice, like github or instagram.
+
+``service``
+    The identifier of the service/website. It's recommended to use the websites
+    domain for this as defined in `Domains`_.
+``username``
+    The users unique identifier, like the login name. If the login name is not
+    known or the system doesn't use login names, use the email address instead.
+``displayname``
+    The users display name. This name is often not unique and may contain the
+    users real name.
+``email``
+    The email address associated with the account.
+``url``
+    The url of the public profile if available.
+``last_seen``
+    The last time this account has been active/online.
+``birthday``
+    The users birthday set on the account.
+``phonenumber``
+    The phonenumber associated with the account.
+``profile_pic``
+    The blob identifier of the users current profile picture.
+
+Breaches
+--------
+
+Either a breach of a specific website, a breach compilation or a breach
+notification service.
+
+``value``
+    The name of the breach, breach compilation or notification service.
+
+Images
+------
+
+``value``
+    The id that identifies the blob. This id is deterministic based on file
+    content.
+``filename``
+    This field is used if we have a well known filename for the content.
+``mime``
+    The image mimetype, like ``image/png`` or ``image/jpeg``.
+``width``
+    The width of the image.
+``height``
+    The height of the image.
+``created``
+    The date and time this image has been taken.
+``latitude``
+    Latitude this picture has been taken.
+``longitude``
+    Longitude this picture has been taken.
+``nudity``
+    A score that classifies nudity in this picture. The score goes from 0 to 2
+    and is commonly calculated with ``img_nudity``. A score above 1 means
+    nudity has been detected.
+``ahash``
+    The Mean (aHash) perceptual hash.
+``dhash``
+    The Gradient (dHash) perceptual hash.
+``phash``
+    The DCT (pHash) perceptual hash.
+
+Ports
+-----
+
+The status of a port on an ip address.
+
+``ip_addr_id``
+    The numeric id of an ipaddr struct.
+``ip_addr``
+    The actual ipaddr.
+``port``
+    The port number.
+``status``
+    The status of the port, either ``open`` or ``closed``.
+``banner``
+    The service banner we discovered on this port.
+``service``
+    The service that is running on this port.
+``version``
+    The version of the service running on this port.
+
+Netblocks
+---------
+
+A netblock is a network address range that has been allocated to an individual,
+organization or company. Those are commonly found when running whois lookups on
+an ip address.
+
+Consider the following example: Running a whois lookup on ``140.82.118.4`` (one
+of the addresses currently in use by github) returns that this address belongs
+to the netrange ``140.82.112.0 - 140.82.127.255``, so the netblock in this case
+is ``140.82.112.0/20``.
+
+``family``
+    This is either ``4`` or ``6`` and populated automatically.
+``value``
+    This is the network range in CIDR notation.
+``asn``
+    The number of the autonomous system this network belongs to.
+``as_org``
+    The organization of the autonomous system this network belongs to.
+``description``
+    This field isn't strictly defined and meant to be used as a human
+    meaningful name if available.
+
+Relations
+---------
+
+Relations are linking two structs together. The link may contain additional information.
+
+subdomain_ipaddr
+~~~~~~~~~~~~~~~~
+
+Links an ip address to a subdomain.
+
+``subdomain_id``
+    The numeric id of a subdomain struct.
+``ip_addr_id``
+    The numeric id of an ip addr struct.
+
+network_device
+~~~~~~~~~~~~~~
+
+Links a device to a network. This is commonly used with ``db_add_ttl`` so the
+link automatically expires. This is frequently used to monitor networks for
+known and unknown devices.
+
+``network_id``
+    The numeric id of a network struct.
+``device_id``
+    The numeric id of a device struct.
+``ipaddr``
+    The ip address assigned to the device.
+``last_seen``
+    The last time we've seen the device on that network.
+
+breach_email
+~~~~~~~~~~~~
+
+Links an email to a breach. If we know the password as well we can add it to
+the link. If we don't know the password we can leave it blank and fill it
+later. An email can be linked to a breach multiple times with different
+passwords. There is a special upserting logic in place to support this.
+
+``breach_id``
+    The numeric id of a breach struct.
+``email_id``
+    The numeric id of an email struct.
+``password``
+    The password for that email in the breach.

docs/usage.rst

@@ -0,0 +1,340 @@
+Running your first investigation
+================================
+
+This page is going to guide you through the process of setting up your
+environment and running your first investigation.
+
+Installing the default modules
+------------------------------
+
+By default, sn0int doesn't have any modules installed. If you start up sn0int
+it's going to download some files that it needs and then suggests to install a
+number of recommended modules::
+
+    $ sn0int
+
+                       ___/           .
+         ____ , __   .'  /\ ` , __   _/_
+        (     |'  `. |  / | | |'  `.  |
+        `--.  |    | |,'  | | |    |  |
+       \___.' /    | /`---' / /    |  \__/
+
+            osint | recon | security
+          irc.hackint.org:6697/#sn0int
+
+    [+] Connecting to database
+    [+] Downloading public suffix list
+    [+] Downloading "GeoLite2-City.mmdb"
+    [+] Downloading "GeoLite2-ASN.mmdb"
+    [+] Loaded 0 modules
+    [*] No modules found, run quickstart to install default modules
+    [sn0int][default] >
+
+Typing ``quickstart`` is going to get you a fair number of featured modules::
+
+    [sn0int][default] > quickstart
+    [+] Installing kpcyrd/asn
+    [+] Installing kpcyrd/ctlogs
+    [+] Installing kpcyrd/dns-resolve
+    [+] Installing kpcyrd/geoip
+    [+] Installing kpcyrd/hackertarget-subdomains
+    [+] Installing kpcyrd/otx-subdomains
+    [+] Installing kpcyrd/passive-spider
+    [+] Installing kpcyrd/pgp-keyserver
+    [+] Installing kpcyrd/threatminer-ipaddr
+    [+] Installing kpcyrd/threatminer-subdomains
+    [+] Installing kpcyrd/url-scan
+    [+] Installing kpcyrd/waybackurls
+    [+] Loaded 12 modules
+    [sn0int][default] >
+
+Adding something to scope
+-------------------------
+
+You probably want to separate your investigations so you should select a
+workspace where your results should go::
+
+    [sn0int][default] > workspace demo
+    [+] Connecting to database
+    [sn0int][demo] >
+
+Next, we have to start somewhere and add the first entity to our scope::
+
+    [sn0int][demo] > add domain
+    Domain: example.com
+    [sn0int][demo] >
+
+.. note::
+   There is a concept of a domain vs a subdomain. We are referring to a domain
+   as everything that is a subdomain of a `public suffix`_. For example, .com
+   is a public suffix, which makes example.com a domain in sn0int terms. Every
+   subdomain of that, like www.example.com, is referred to as a subdomain.
+
+   Note that example.com can be added as a subdomain as well since it can hold
+   records. In that case, example.com is both the name of the dns zone, while
+   also being an entity in that zone.
+
+.. _public suffix: https://publicsuffix.org/
+
+You can confirm this by running a select on the domains we now have::
+
+    [sn0int][demo] > select domains
+    #1, "example.com"
+    [sn0int][demo] >
+
+Something we don't need right now, but is going to be useful later on is the
+ability to filter your entities::
+
+    [sn0int][demo] > select domains where id=1
+    #1, "example.com"
+    [sn0int][demo] >
+    [sn0int][demo] > select domains where value like %.com
+    #1, "example.com"
+    [sn0int][demo] >
+    [sn0int][demo] > select domains where ( value like e% and value like %m ) or false
+    #1, "example.com"
+    [sn0int][demo] >
+
+.. note::
+   Almost all entities have a ``value`` column that holds the primary value of
+   the entity.
+
+Running a module
+----------------
+
+Now that we have something to get started with, we can run our first module.
+First lets list all modules we have::
+
+    [sn0int][demo] > mod list
+    kpcyrd/asn (0.1.0)
+        Run a asn lookup for an ip address
+    kpcyrd/ctlogs (0.1.0)
+        Query certificate transparency logs to discover subdomains
+    kpcyrd/dns-resolve (0.1.0)
+        Query subdomains to discovery ip addresses and verify the record is visible
+    kpcyrd/geoip (0.1.0)
+        Run a geoip lookup for an ip address
+    kpcyrd/hackertarget-subdomains (0.1.0)
+        Query hackertarget for subdomains of a domain
+    kpcyrd/otx-subdomains (0.1.0)
+        Query alienvault otx passive dns for subdomains of a domain
+    kpcyrd/passive-spider (0.1.0)
+        Scrape known http responses for urls
+    kpcyrd/pgp-keyserver (0.1.0)
+        Query pgp keyserver for email addresses
+    kpcyrd/threatminer-ipaddr (0.1.0)
+        Query ThreatMiner passive dns for subdomains of an ip address
+    kpcyrd/threatminer-subdomains (0.1.0)
+        Query ThreatMiner passive dns for subdomains of a domain
+    kpcyrd/url-scan (0.1.0)
+        Scan subdomains for websites
+    kpcyrd/waybackurls (0.1.0)
+        Discover subdomains from wayback machine
+    [sn0int][demo] >
+
+Let's start by querying certificate transparency logs::
+
+    [sn0int][demo] > use ctlogs
+    [sn0int][demo][kpcyrd/ctlogs] > run
+    [*] "example.com"                                     : Subdomain: "www.example.com"
+    [*] "example.com"                                     : Subdomain: "m.example.com"
+    [*] "example.com"                                     : Subdomain: "dev.example.com"
+    [*] "example.com"                                     : Subdomain: "products.example.com"
+    [*] "example.com"                                     : Subdomain: "support.example.com"
+    [+] Finished kpcyrd/ctlogs
+    [sn0int][demo][kpcyrd/ctlogs] >
+
+Looks like we've discovered some subdomains here. It might be tempting to throw
+some of them in a browser but hold on, there's a more efficient way to approach
+this.
+
+.. hint::
+   You can run the modules concurrently with ``run -j 8``.
+
+Running followup modules on the results
+---------------------------------------
+
+A lot of time has been spent on the database part. While it sort of feels like
+a no-sql database we are actually enforcing a schema for a reason instead of
+just using generic dictionaries and calling it a day.
+
+It's crucial that entities created by one module can be picked up by another
+module, like LEGOs. Let's continue with a module to query the dns records::
+
+    [sn0int][demo][kpcyrd/ctlogs] > use dns-resolve
+    [sn0int][demo][kpcyrd/dns-resolve] > run
+    [*] "www.example.com"                                 : Updating "www.example.com" (resolvable => true)
+    [*] "www.example.com"                                 : IpAddr: 93.184.216.34
+    [*] "www.example.com"                                 : "www.example.com" -> 93.184.216.34
+    [*] "m.example.com"                                   : Updating "m.example.com" (resolvable => false)
+    [*] "dev.example.com"                                 : Updating "dev.example.com" (resolvable => false)
+    [*] "products.example.com"                            : Updating "products.example.com" (resolvable => false)
+    [*] "support.example.com"                             : Updating "support.example.com" (resolvable => false)
+    [+] Finished kpcyrd/dns-resolve
+    [sn0int][demo][kpcyrd/dns-resolve] >
+
+.. TODO: mention https://github.com/kpcyrd/sn0int/issues/27
+
+Two things happened here: We've discovered some IP addresses and added them to
+scope, and we also updated our subdomain entities with new information, since
+we now know which of them are resolvable and which aren't.
+
+Let's run the next module, which is actually going to check for websites on
+them, but let's only target subdomains that we know are resolvable::
+
+    [sn0int][demo][kpcyrd/dns-resolve] > use url-scan
+    [sn0int][demo][kpcyrd/url-scan] > target
+    #1, "www.example.com"
+        93.184.216.34
+    #2, "m.example.com"
+    #3, "dev.example.com"
+    #4, "products.example.com"
+    #5, "support.example.com"
+    [sn0int][demo][kpcyrd/url-scan] > target where resolvable
+    [+] 1 entities selected
+    [sn0int][demo][kpcyrd/url-scan] > target
+    #1, "www.example.com"
+        93.184.216.34
+    [sn0int][demo][kpcyrd/url-scan] >
+
+We can both preview and limit the targets that are going to be passed to the
+module with the target command. Once we are satisfied with our selection we can
+run this module::
+
+    [sn0int][demo][kpcyrd/url-scan] > run
+    [*] "www.example.com"                                 : Url: "http://www.example.com/" (200)
+    [*] "www.example.com"                                 : Url: "https://www.example.com/" (200)
+    [+] Finished kpcyrd/url-scan
+    [sn0int][demo][kpcyrd/url-scan] >
+
+We've now probed both port 80 and port 443 for each subdomain and found two
+http responses this way. If you want a list of urls you may want to visit in
+your browser can now query them::
+
+    [sn0int][demo][kpcyrd/url-scan] > select urls
+    #1, "http://www.example.com/" (200)
+    #2, "https://www.example.com/" (200)
+    [sn0int][demo][kpcyrd/url-scan] >
+
+Unscoping entities
+------------------
+
+Something you are going to run into is that modules are too greedy and add
+things to the scope we are not interested in. You can delete them using the
+delete command, but those are likely picked up by a module again.
+
+What you can do instead is setting a flag on an entity that removes it from
+our scope. This is done using the noscope command::
+
+    [sn0int][demo] > use ctlogs
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    [sn0int][demo][kpcyrd/ctlogs] > add domain
+    Domain: google.com
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    #2, "google.com"
+    [sn0int][demo][kpcyrd/ctlogs] > noscope domains where value=google.com
+    [+] Updated 1 rows
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    [sn0int][demo][kpcyrd/ctlogs] >
+
+Entities that are unscoped are automatically ignored by all modules.
+
+You can reverse this using the scope command::
+
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    [sn0int][demo][kpcyrd/ctlogs] > scope domains where true
+    [+] Updated 2 rows
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    #2, "google.com"
+    [sn0int][demo][kpcyrd/ctlogs] >
+
+.. hint::
+   All entities have this field, you can refer to it in queries using
+   ``unscoped=1``.
+
+Autonoscope
+-----------
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+~~~~~~~
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+~~~
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+~~~~
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

examples/maxmind.rs

@@ -0,0 +1,104 @@
+extern crate sn0int;
+extern crate env_logger;
+extern crate chrootable_https;
+#[macro_use] extern crate log;
+
+// workaround for rustc 1.29.2 support
+#[cfg(not(target_os = "openbsd"))]
+extern crate structopt;
+#[cfg(target_os = "openbsd")]
+#[macro_use] extern crate structopt;
+
+use sn0int::errors::*;
+use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
+use sn0int::paths;
+use std::fs;
+use std::net::IpAddr;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="dl")]
+    Download(Download),
+    #[structopt(name="asn")]
+    Asn(AsnArgs),
+    #[structopt(name="geoip")]
+    GeoIP(GeoIPArgs),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Download {
+    url: String,
+    filter: String,
+    target: String,
+    #[structopt(short="e", long="extract-only")]
+    extract_only: bool,
+}
+
+impl Download {
+    fn run(&self) -> Result<()> {
+        let path = paths::cache_dir()?.join(&self.target);
+        if self.extract_only {
+            let body = fs::read(&self.url)?;
+            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
+        } else {
+            GeoIP::download(path, &self.filter, &self.url)?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AsnArgs {
+    ip: IpAddr,
+}
+
+impl AsnArgs {
+    fn run(&self) -> Result<()> {
+        let asndb = AsnDB::open_or_download()?;
+
+        let asn = asndb.lookup(self.ip)?;
+        println!("{:#?}", asn);
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct GeoIPArgs {
+    ip: IpAddr,
+}
+
+impl GeoIPArgs {
+    fn run(&self) -> Result<()> {
+        let geoip = GeoIP::open_or_download()?;
+
+        let lookup = geoip.lookup(self.ip)?;
+        println!("{:#?}", lookup);
+
+        Ok(())
+    }
+}
+
+
+fn run() -> Result<()> {
+    let args = Args::from_args();
+    debug!("{:?}", args);
+    match args {
+        Args::Download(args) => args.run(),
+        Args::Asn(args) => args.run(),
+        Args::GeoIP(args) => args.run(),
+    }
+}
+
+fn main() {
+    env_logger::init();
+
+    if let Err(err) = run() {
+        eprintln!("Error: {}", err);
+        for cause in err.iter_chain().skip(1) {
+            eprintln!("Because: {}", cause);
+        }
+        std::process::exit(1);
+    }
+}

examples/spinners.rs

@@ -1,20 +1,67 @@
-extern crate sn0int;
-
-use std::env;
 use std::thread;
 use std::time::Duration;
-use sn0int::term::{SPINNERS, Spinner};
+use sn0int::term::{SPINNERS, Spinner, StackedSpinners};
+use structopt::StructOpt;
+
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="single")]
+    Single(Single),
+    #[structopt(name="stacked")]
+    Stacked(Stacked),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Single {
+    idx: usize,
+    #[structopt(long="ticks", default_value="100")]
+    ticks: usize,
+}
+
+impl Single {
+    fn run(&self) {
+        let mut s = Spinner::new(SPINNERS[self.idx], "Demo".to_string());
+
+        for _ in 0..self.ticks {
+            thread::sleep(Duration::from_millis(100));
+            s.tick();
+        }
+
+        s.finish("Done".to_string());
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Stacked {
+}
+
+impl Stacked {
+    fn run(&self) {
+        let mut stack = StackedSpinners::new();
+        stack.add("1".into(), String::from("spinner1"));
+        stack.add("2".into(), String::from("spinner2"));
+        stack.add("3".into(), String::from("spinner3"));
+
+        for x in 1..=3 {
+            for _ in 0..50 {
+                thread::sleep(Duration::from_millis(100));
+                stack.tick();
+            }
+            // stack.log("ohai");
+            stack.remove(&x.to_string());
+        }
+
+        stack.clear();
+
+        // stack.finish("Done".to_string());
+    }
+}
 
 fn main() {
-    let idx = env::args().skip(1).next().expect("Expected argv[1]");
-    let idx = idx.parse::<usize>().expect("argv[1] is not a number");
-
-    let mut s = Spinner::new(SPINNERS[idx], "Demo".to_string());
-
-    for _ in 0..100 {
-        thread::sleep(Duration::from_millis(100));
-        s.tick();
+    let args = Args::from_args();
+    match args {
+        Args::Single(args) => args.run(),
+        Args::Stacked(args) => args.run(),
     }
-
-    s.finish("Done".to_string());
 }

migrations/2018-09-12-102647_create_domains/up.sql

@@ -24,7 +24,7 @@ CREATE TABLE subdomain_ipaddrs (
     id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
     subdomain_id INTEGER NOT NULL,
     ip_addr_id INTEGER NOT NULL,
-    FOREIGN KEY(subdomain_id) REFERENCES domains(id),
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
     FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id),
     CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
 );
@@ -35,6 +35,6 @@ CREATE TABLE urls (
     value VARCHAR NOT NULL,
     status INTEGER,
     body BLOB,
-    FOREIGN KEY(subdomain_id) REFERENCES domains(id),
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
     CONSTRAINT url_unique UNIQUE (value)
 );

migrations/2018-10-12-051052_emails/down.sql

@@ -0,0 +1 @@
+DROP TABLE emails;

migrations/2018-10-12-051052_emails/up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    CONSTRAINT email_unique UNIQUE (value)
+);

migrations/2018-10-15-061351_scope/down.sql

@@ -0,0 +1,90 @@
+PRAGMA foreign_keys=off;
+
+-- domains
+
+ALTER TABLE domains RENAME TO _domains_old;
+
+CREATE TABLE domains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    CONSTRAINT domain_unique UNIQUE (value)
+);
+
+INSERT INTO domains (id, value)
+  SELECT id, value
+  FROM _domains_old;
+
+DROP TABLE _domains_old;
+
+-- subdomains
+
+ALTER TABLE subdomains RENAME TO _subdomains_old;
+
+CREATE TABLE subdomains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    domain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    FOREIGN KEY(domain_id) REFERENCES domains(id),
+    CONSTRAINT subdomain_unique UNIQUE (value)
+);
+
+INSERT INTO subdomains (id, domain_id, value)
+  SELECT id, domain_id, value
+  FROM _subdomains_old;
+
+DROP TABLE _subdomains_old;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value)
+  SELECT id, family, value
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+-- urls
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body)
+  SELECT id, subdomain_id, value, status, body
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+-- emails
+
+ALTER TABLE emails RENAME TO _emails_old;
+
+CREATE TABLE emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    CONSTRAINT domain_unique UNIQUE (value)
+);
+
+INSERT INTO emails (id, value)
+  SELECT id, value
+  FROM _emails_old;
+
+DROP TABLE _emails_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-10-15-061351_scope/up.sql

@@ -0,0 +1,5 @@
+ALTER TABLE domains ADD COLUMN unscoped BOOLEAN DEFAULT 0 NOT NULL;
+ALTER TABLE subdomains ADD COLUMN unscoped BOOLEAN DEFAULT 0 NOT NULL;
+ALTER TABLE ipaddrs ADD COLUMN unscoped BOOLEAN DEFAULT 0 NOT NULL;
+ALTER TABLE urls ADD COLUMN unscoped BOOLEAN DEFAULT 0 NOT NULL;
+ALTER TABLE emails ADD COLUMN unscoped BOOLEAN DEFAULT 0 NOT NULL;

migrations/2018-10-17-112220_verifiable/down.sql

@@ -0,0 +1,60 @@
+PRAGMA foreign_keys=off;
+
+-- subdomains
+
+ALTER TABLE subdomains RENAME TO _subdomains_old;
+
+CREATE TABLE subdomains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    domain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    FOREIGN KEY(domain_id) REFERENCES domains(id),
+    CONSTRAINT subdomain_unique UNIQUE (value)
+);
+
+INSERT INTO subdomains (id, domain_id, value, unscoped)
+  SELECT id, domain_id, value, unscoped
+  FROM _subdomains_old;
+
+DROP TABLE _subdomains_old;
+
+-- emails
+
+ALTER TABLE emails RENAME TO _emails_old;
+
+CREATE TABLE emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT emails_unique UNIQUE (value)
+);
+
+INSERT INTO emails (id, value, unscoped)
+  SELECT id, value, unscoped
+  FROM _emails_old;
+
+DROP TABLE _emails_old;
+
+-- urls
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped)
+  SELECT id, subdomain_id, value, status, body, unscoped
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-10-17-112220_verifiable/up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE subdomains ADD COLUMN resolvable BOOLEAN;
+ALTER TABLE emails ADD COLUMN valid BOOLEAN;
+ALTER TABLE urls ADD COLUMN online BOOLEAN;

migrations/2018-10-22-141819_geoip/down.sql

@@ -0,0 +1,21 @@
+PRAGMA foreign_keys=off;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value, unscoped)
+  SELECT id, family, value, unscoped
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-10-22-141819_geoip/up.sql

@@ -0,0 +1,7 @@
+ALTER TABLE ipaddrs ADD COLUMN continent VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN continent_code VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN country VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN country_code VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN city VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN latitude FLOAT;
+ALTER TABLE ipaddrs ADD COLUMN longitude FLOAT;

migrations/2018-10-24-131856_asn/down.sql

@@ -0,0 +1,28 @@
+PRAGMA foreign_keys=off;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude)
+  SELECT id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-10-24-131856_asn/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ipaddrs ADD COLUMN asn INTEGER;
+ALTER TABLE ipaddrs ADD COLUMN as_org VARCHAR;

migrations/2018-10-26-001932_cascade-delete/down.sql

@@ -0,0 +1,126 @@
+PRAGMA foreign_keys=off;
+
+-- domains
+
+ALTER TABLE domains RENAME TO _domains_old;
+
+CREATE TABLE domains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT domain_unique UNIQUE (value)
+);
+
+INSERT INTO domains (id, value, unscoped)
+    SELECT id, value, unscoped
+    FROM _domains_old;
+
+DROP TABLE _domains_old;
+
+-- subdomains
+
+ALTER TABLE subdomains RENAME TO _subdomains_old;
+
+CREATE TABLE subdomains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    domain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    resolvable BOOLEAN,
+    FOREIGN KEY(domain_id) REFERENCES domains(id),
+    CONSTRAINT subdomain_unique UNIQUE (value)
+);
+
+INSERT INTO subdomains (id, domain_id, value, unscoped, resolvable)
+    SELECT id, domain_id, value, unscoped, resolvable
+    FROM _subdomains_old;
+
+DROP TABLE _subdomains_old;
+
+-- subdomain_ipaddrs
+
+ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
+
+CREATE TABLE subdomain_ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id),
+    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
+);
+
+INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
+    SELECT id, subdomain_id, ip_addr_id
+    FROM _subdomain_ipaddrs_old;
+
+DROP TABLE _subdomain_ipaddrs_old;
+
+-- urls
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id),
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online)
+    SELECT id, subdomain_id, value, status, body, unscoped, online
+    FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+-- emails
+
+ALTER TABLE emails RENAME TO _emails_old;
+
+CREATE TABLE emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO emails (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM _emails_old;
+
+DROP TABLE _emails_old;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    asn INTEGER,
+    as_org VARCHAR,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude, asn, as_org)
+  SELECT id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude, asn, as_org
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-10-26-001932_cascade-delete/up.sql

@@ -0,0 +1,126 @@
+PRAGMA foreign_keys=off;
+
+-- domains
+
+ALTER TABLE domains RENAME TO _domains_old;
+
+CREATE TABLE domains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT domain_unique UNIQUE (value)
+);
+
+INSERT INTO domains (id, value, unscoped)
+    SELECT id, value, unscoped
+    FROM _domains_old;
+
+DROP TABLE _domains_old;
+
+-- subdomains
+
+ALTER TABLE subdomains RENAME TO _subdomains_old;
+
+CREATE TABLE subdomains (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    domain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    resolvable BOOLEAN,
+    FOREIGN KEY(domain_id) REFERENCES domains(id) ON DELETE CASCADE,
+    CONSTRAINT subdomain_unique UNIQUE (value)
+);
+
+INSERT INTO subdomains (id, domain_id, value, unscoped, resolvable)
+    SELECT id, domain_id, value, unscoped, resolvable
+    FROM _subdomains_old;
+
+DROP TABLE _subdomains_old;
+
+-- urls
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online)
+    SELECT id, subdomain_id, value, status, body, unscoped, online
+    FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+-- emails
+
+ALTER TABLE emails RENAME TO _emails_old;
+
+CREATE TABLE emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO emails (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM _emails_old;
+
+DROP TABLE _emails_old;
+
+-- ipaddrs
+
+ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
+
+CREATE TABLE ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    asn INTEGER,
+    as_org VARCHAR,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO ipaddrs (id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude, asn, as_org)
+  SELECT id, family, value, unscoped, continent, continent_code, country, country_code, city, latitude, longitude, asn, as_org
+  FROM _ipaddrs_old;
+
+DROP TABLE _ipaddrs_old;
+
+-- subdomain_ipaddrs
+
+ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
+
+CREATE TABLE subdomain_ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
+);
+
+INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
+    SELECT id, subdomain_id, ip_addr_id
+    FROM _subdomain_ipaddrs_old;
+
+DROP TABLE _subdomain_ipaddrs_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-11-03-230746_extend-urls/down.sql

@@ -0,0 +1,25 @@
+PRAGMA foreign_keys=off;
+
+-- urls
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online)
+    SELECT id, subdomain_id, value, status, body, unscoped, online
+    FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-11-03-230746_extend-urls/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE urls ADD COLUMN title VARCHAR;
+ALTER TABLE urls ADD COLUMN redirect VARCHAR;

migrations/2018-12-03-143558_url_path/down.sql

@@ -0,0 +1,25 @@
+PRAGMA foreign_keys=off;
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    title VARCHAR,
+    redirect VARCHAR,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online, title, redirect)
+  SELECT id, subdomain_id, value, status, body, unscoped, online, title, redirect
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-03-143558_url_path/up.sql

@@ -0,0 +1,26 @@
+PRAGMA foreign_keys=off;
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    path VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    title VARCHAR,
+    redirect VARCHAR,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, path, status, body, unscoped, online, title, redirect)
+  SELECT id, subdomain_id, value, '/', status, body, unscoped, online, title, redirect
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-14-094011_phonenumbers/down.sql

@@ -0,0 +1 @@
+DROP TABLE phonenumbers;

migrations/2018-12-14-094011_phonenumbers/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE phonenumbers (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    name VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    last_online DATETIME,
+    country VARCHAR,
+    carrier VARCHAR,
+    line VARCHAR,
+    is_ported BOOLEAN,
+    last_ported DATETIME,
+    caller_name VARCHAR,
+    caller_type VARCHAR,
+    CONSTRAINT phonenumber_unique UNIQUE (value)
+);

migrations/2018-12-23-230955_reverse-dns/down.sql

@@ -0,0 +1,27 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ipaddrs_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    asn INTEGER,
+    as_org VARCHAR,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO _ipaddrs_new (id, family, value, unscoped, continent, continent_code, city, latitude, longitude, asn, as_org)
+  SELECT id, family, value, unscoped, continent, continent_code, city, latitude, longitude, asn, as_org
+  FROM ipaddrs;
+
+DROP TABLE ipaddrs;
+ALTER TABLE _ipaddrs_new RENAME TO ipaddrs;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-23-230955_reverse-dns/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ipaddrs ADD COLUMN description VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN reverse_dns VARCHAR;

migrations/2018-12-24-141533_networks/down.sql

@@ -0,0 +1,3 @@
+DROP TABLE network_devices;
+DROP TABLE networks;
+DROP TABLE devices;

migrations/2018-12-24-141533_networks/up.sql

@@ -0,0 +1,30 @@
+CREATE TABLE networks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+CREATE TABLE devices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    name VARCHAR,
+    hostname VARCHAR,
+    vendor VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    last_seen DATETIME,
+    CONSTRAINT device_unique UNIQUE (value)
+);
+
+CREATE TABLE network_devices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    network_id INTEGER NOT NULL,
+    device_id INTEGER NOT NULL,
+    ipaddr VARCHAR,
+    last_seen DATETIME,
+    FOREIGN KEY(network_id) REFERENCES networks(id) ON DELETE CASCADE,
+    FOREIGN KEY(device_id) REFERENCES devices(id) ON DELETE CASCADE,
+    CONSTRAINT network_device_unique UNIQUE (network_id, device_id)
+);

migrations/2019-01-20-000235_ttl/down.sql

@@ -0,0 +1 @@
+DROP TABLE ttls;

migrations/2019-01-20-000235_ttl/up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE ttls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);

migrations/2019-02-03-123424_accounts/down.sql

@@ -0,0 +1 @@
+DROP TABLE accounts;

migrations/2019-02-03-123424_accounts/up.sql

@@ -0,0 +1,12 @@
+CREATE TABLE accounts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);

migrations/2019-02-11-011316_breaches/down.sql

@@ -0,0 +1,2 @@
+DROP TABLE breach_emails;
+DROP TABLE breaches;

migrations/2019-02-11-011316_breaches/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE breaches (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT breach_unique UNIQUE (value)
+);
+
+CREATE TABLE breach_emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    breach_id INTEGER NOT NULL,
+    email_id INTEGER NOT NULL,
+    password VARCHAR,
+    FOREIGN KEY(breach_id) REFERENCES breaches(id) ON DELETE CASCADE,
+    FOREIGN KEY(email_id) REFERENCES emails(id) ON DELETE CASCADE,
+    CONSTRAINT breach_emails_unique UNIQUE (breach_id, email_id, password)
+);

migrations/2019-03-08-060037_images/down.sql

@@ -0,0 +1 @@
+DROP TABLE images;

migrations/2019-03-08-060037_images/up.sql

@@ -0,0 +1,21 @@
+CREATE TABLE images (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+
+    filename VARCHAR,
+    mime VARCHAR,
+    width INT,
+    height INT,
+    created DATETIME,
+
+    latitude FLOAT,
+    longitude FLOAT,
+
+    nudity FLOAT,
+    ahash VARCHAR,
+    dhash VARCHAR,
+    phash VARCHAR,
+
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT image_unique UNIQUE (value)
+);

migrations/2019-03-24-195306_email-names/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _emails_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO _emails_new (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM emails;
+
+DROP TABLE emails;
+ALTER TABLE _emails_new RENAME TO emails;
+
+PRAGMA foreign_keys=on;

migrations/2019-03-24-195306_email-names/up.sql

@@ -0,0 +1,2 @@
+-- Your SQL goes here
+ALTER TABLE emails ADD COLUMN displayname VARCHAR;

migrations/2019-05-30-142142_ports/down.sql

@@ -0,0 +1 @@
+DROP TABLE ports;

migrations/2019-05-30-142142_ports/up.sql

@@ -0,0 +1,17 @@
+CREATE TABLE ports (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);

migrations/2019-07-04-233515_autonoscope/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE autonoscope;

migrations/2019-07-04-233515_autonoscope/up.sql

@@ -0,0 +1,8 @@
+-- Your SQL goes here
+CREATE TABLE autonoscope (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    object VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    scoped BOOLEAN NOT NULL,
+    CONSTRAINT autonoscope_unique UNIQUE (object, value)
+);

migrations/2019-07-27-152215_netblocks/down.sql

@@ -0,0 +1 @@
+DROP TABLE netblocks;

migrations/2019-07-27-152215_netblocks/up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE netblocks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    asn INTEGER,
+    as_org VARCHAR,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2019-08-11-073709_misc-fields/down.sql

@@ -0,0 +1,41 @@
+PRAGMA foreign_keys=off;
+
+-- accounts
+CREATE TABLE _accounts_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);
+
+INSERT INTO _accounts_new (id, value, service, username, displayname, email, url, last_seen, unscoped)
+    SELECT id, value, service, username, displayname, email, url, last_seen, unscoped
+    FROM accounts;
+
+DROP TABLE accounts;
+ALTER TABLE _accounts_new RENAME TO accounts;
+
+-- networks
+CREATE TABLE _networks_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+INSERT INTO _networks_new (id, value, unscoped, latitude, longitude)
+    SELECT id, value, unscoped, latitude, longitude
+    FROM networks;
+
+DROP TABLE networks;
+ALTER TABLE _networks_new RENAME TO networks;
+
+PRAGMA foreign_keys=on;

migrations/2019-08-11-073709_misc-fields/up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE accounts ADD COLUMN phonenumber VARCHAR;
+ALTER TABLE accounts ADD COLUMN profile_pic VARCHAR;
+ALTER TABLE accounts ADD COLUMN birthday VARCHAR;
+ALTER TABLE networks ADD COLUMN description VARCHAR;

modules/dev/54267.lua

@@ -1,12 +0,0 @@
--- Description: Reproduce a rust stdlib panic
--- Version: 0.1.0
--- Source: domains
-
-function run()
-    --[[
-    See for details:
-    https://github.com/rust-lang/rust/issues/54267
-    https://github.com/rust-lang/rust/issues/39364
-    ]]--
-    sleep(10)
-end

modules/dev/apex-domain.lua

@@ -0,0 +1,11 @@
+-- Description: Create subdomain entries for apex domains
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    db_add('subdomain', {
+        domain_id=arg['id'],
+        value=arg['value'],
+    })
+end

modules/dev/arp-scan.lua

@@ -0,0 +1,45 @@
+-- Description: Parse arp-scan output
+-- Version: 0.3.0
+-- License: GPL-3.0
+
+-- sudo arp-scan -qglI wlp3s0
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        m = regex_find('(.+)\t(.+)', x)
+        if m ~= nil then
+            ipaddr = m[2]
+            mac = m[3]
+            now = datetime()
+
+            device_id = db_add('device', {
+                value=mac,
+                last_seen=now,
+            })
+            if last_err() then return end
+
+            db_add_ttl('network-device', {
+                network_id=network_id,
+                device_id=device_id,
+                ipaddr=ipaddr,
+                last_seen=now,
+            }, 300)
+            if last_err() then return end
+        end
+    end
+end

modules/dev/asn.lua

@@ -0,0 +1,16 @@
+-- Description: Run a asn lookup for an ip address
+-- Version: 0.1.0
+-- Source: ipaddrs
+-- License: GPL-3.0
+
+function run(arg)
+    lookup = asn_lookup(arg['value'])
+    if last_err() then return end
+
+    if arg['asn'] ~= lookup['asn'] or arg['as_org'] ~= lookup['as_org'] then
+        db_update('ipaddr', arg, {
+            asn=lookup['asn'],
+            as_org=lookup['as_org'],
+        })
+    end
+end

modules/dev/axfr.lua

@@ -0,0 +1,145 @@
+-- Description: Try a zone transfer for subdomains
+-- Version: 0.3.0
+-- Source: domains
+-- License: GPL-3.0
+
+function strip_root_dot(name)
+    local m = regex_find("(.+)\\.$", name)
+    if last_err() then return end
+
+    if m == nil then
+        return name
+    else
+        return m[2]
+    end
+end
+
+function add_pointer(name)
+    -- select psl+1
+    local domain = psl_domain_from_dns_name(name)
+    if last_err() then return end
+
+    -- add domain
+    local domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    -- add subdomain
+    local subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+    if last_err() then return end
+end
+
+function iter_axfr(zone, arg)
+    debug(arg)
+
+    local name = arg[1]
+    local r = arg[2]
+
+    -- select psl+1
+    local domain = psl_domain_from_dns_name(name)
+    if last_err() then return end
+
+    -- add domain
+    local domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    -- add subdomain
+    local subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+    if last_err() then return end
+
+    -- this is a A record
+    if r['A'] ~= nil then
+        -- add the name and ip
+        local ipaddr_id = db_add('ipaddr', {
+            family='4',
+            value=r['A'],
+        })
+        if last_err() then return end
+
+        db_add('subdomain-ipaddr', {
+            subdomain_id=subdomain_id,
+            ip_addr_id=ipaddr_id,
+        })
+        if last_err() then return end
+    end
+
+    if r['CNAME'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['CNAME'])
+        add_pointer(name)
+    end
+
+    if r['NS'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['NS'])
+        add_pointer(name)
+    end
+
+    if r['MX'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['MX'][2])
+        add_pointer(name:lower())
+    end
+end
+
+function iter_a(zone, arg)
+    if arg == nil then return end
+
+    debug('nameserver: ' .. arg)
+    local records = dns(zone, {
+        record='AXFR',
+        nameserver=arg .. ':53',
+        tcp=true,
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    for i=1, #records do
+        iter_axfr(zone, records[i])
+        if last_err() then return end
+    end
+end
+
+function iter_ns(zone, arg)
+    if arg == nil then return end
+
+    local records = dns(arg, {
+        record='A',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    for i=1, #records do
+        r = records[i][2]
+        iter_a(zone, r['A'])
+        if last_err() then return end
+    end
+end
+
+function run(arg)
+    local records = dns(arg['value'], {
+        record='NS',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    for i=1, #records do
+        local r = records[i][2]
+        iter_ns(arg['value'], r['NS'])
+        if last_err() then return end
+    end
+end

modules/dev/chaturbate.lua

@@ -0,0 +1,66 @@
+-- Description: Collect information from chaturbate streams
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: accounts:chaturbate.com
+
+function resolve_external_link(href)
+    local link = url_join('https://chaturbate.com/', href)
+    local parts = url_parse(link)
+    return parts['params']['url']
+end
+
+service_map = {}
+service_map['www.instagram.com']    = {'^/([^/]+)', 'instagram.com'}
+service_map['twitter.com']          = {'^/([^/]+)', 'twitter.com'}
+service_map['www.patreon.com']      = {'^/([^/]+)', 'patreon.com'}
+service_map['www.twitch.tv']        = {'^/([^/]+)', 'twitch.tv'}
+
+function detect_account(link)
+    local parts = url_parse(link)
+    if last_err() then return clear_err() end
+    local host = parts['host']
+
+    local service = service_map[host]
+    if service then
+        local m = regex_find(service[1], parts['path'])
+        if m then
+            db_add('account', {
+                service=service[2],
+                username=m[2],
+            })
+        end
+    end
+end
+
+function run(arg)
+    local session = http_mksession()
+
+    local url = 'https://chaturbate.com/' .. arg['username'] .. '/'
+    local req = http_request(session, 'GET', url, {})
+    local r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then
+        return 'http error: ' .. r['status']
+    end
+
+    local last_seen = nil
+    if #html_select_list(r['text'], '.offline_tipping') > 0 then
+        debug('is offline')
+    else
+        debug('is online')
+        last_seen = datetime()
+    end
+
+    db_update('account', arg, {
+        last_seen=last_seen,
+        url='https://chaturbate.com/' .. arg['username'] .. '/',
+    })
+
+    local links = html_select_list(r['text'], 'a[href^="/external_link/"]')
+    for i=1, #links do
+        local link = resolve_external_link(links[i]['attrs']['href'])
+        debug(link)
+        detect_account(link)
+        if last_err() then return end
+    end
+end

modules/dev/cname-harvest.lua

@@ -0,0 +1,48 @@
+-- Description: Query for CNAMES to find subdomains
+-- Version: 0.3.0
+-- Source: subdomains
+-- License: GPL-3.0
+
+function iter(r)
+    if r == nil then
+        return
+    end
+
+    m = regex_find("(.+)\\.$", r)
+    if last_err() then return end
+
+    if m == nil then
+        return
+    end
+    r = m[2]
+
+    domain = psl_domain_from_dns_name(r)
+    if last_err() then return end
+
+    domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+
+    if domain_id ~= nil then
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=r,
+        })
+        if last_err() then return end
+    end
+end
+
+function run(arg)
+    records = dns(arg['value'], 'A')
+    if last_err() then return end
+
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    for i=1, #records do
+        r = records[i][2]
+        iter(r['CNAME'])
+        if last_err() then return end
+    end
+end

modules/dev/ctlogs.lua

@@ -1,11 +1,59 @@
 -- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.1.0
+-- Version: 0.5.0
 -- Source: domains
+-- License: GPL-3.0
+
+function each_name(name)
+    local domain_id, psl_domain
+
+    if seen[name] == 1 then
+        return
+    end
+    seen[name] = 1
+    debug(name)
+
+    if name:find('*.') == 1 then
+        -- ignore wildcard domains
+        return
+    end
+
+    -- the cert might be valid for subdomains that do not belong to the
+    -- domain we started with
+    psl_domain = psl_domain_from_dns_name(name)
+    domain_id = domains[psl_domain]
+    if domain_id == nil then
+        if any_domain then
+            -- unknown domains should be added to database
+            domain_id = db_add('domain', {
+                value=psl_domain,
+            })
+        else
+            -- only use domains that are already in scope
+            domain_id = db_select('domain', psl_domain)
+        end
+
+        -- if we didn't get a valid id, skip
+        if domain_id == nil then
+            return
+        end
+
+        domains[psl_domain] = domain_id
+    end
+
+    db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+end
 
 function run(arg)
-    session = http_mksession()
+    full = getopt('full') ~= nil
+    any_domain = getopt('any-domain') ~= nil
 
-    -- TODO: example.com needs to be dynamic
+    domains = {}
+    domains[arg['value']] = arg['id']
+
+    session = http_mksession()
     req = http_request(session, 'GET', 'https://crt.sh/', {
         query={
             q='%.' .. arg['value'],
@@ -17,32 +65,37 @@ function run(arg)
     if last_err() then return end
     if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
 
-    certs = json_decode_stream(resp['text'])
+    certs = json_decode(resp['text'])
     if last_err() then return end
 
     seen = {}
 
-    i = 1
-    while i <= #certs do
+    for i=1, #certs do
         c = certs[i]
-        -- print(c)
+        debug(c)
 
-        name = c['name_value']
-
-        if name:find("*.") == 1 then
-            -- ignore wildcard domains
-            seen[name] = 1
-        end
-
-        if seen[name] == nil then
-            -- info(name)
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=name,
+        if full then
+            -- fetch certificate
+            id = c['min_cert_id']
+            req = http_request(session, 'GET', 'https://crt.sh/', {
+                query={
+                    d=id .. '', -- TODO: find nicer way for tostring
+                }
             })
-            seen[name] = 1
-        end
+            resp = http_send(req)
+            if last_err() then return end
+            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
 
-        i = i+1
+            -- iterate over all valid names
+            crt = x509_parse_pem(resp['text'])
+            if last_err() then return end
+            names = crt['valid_names']
+
+            for j=1, #names do
+                each_name(names[j])
+            end
+        else
+            each_name(c['name_value'])
+        end
     end
 end

modules/dev/cve-2014-8244.lua

@@ -0,0 +1,89 @@
+-- Description: Get connected devices from linksys router via JNAP
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+    local network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    local router = getopt('router')
+    if not router then
+        return 'router option is missing (http://192.0.2.1:9000/)'
+    end
+
+    local target = url_join(router, 'JNAP/')
+    debug(target)
+
+    local session = http_mksession()
+    local headers = {}
+    headers['Content-Type'] = 'application/json; charset=UTF-8'
+    headers['X-JNAP-Action'] = 'http://linksys.com/jnap/core/Transaction'
+    headers['X-JNAP-Authorization'] = 'null'
+    local req = http_request(session, 'POST', target, {
+        headers=headers,
+        body='[{"action":"http://linksys.com/jnap/devicelist/GetDevices","request":{"sinceRevision":0}},{"action":"http://linksys.com/jnap/networkconnections/GetNetworkConnections","request":{}}]',
+    })
+    local r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then
+        return 'http error: ' .. r['status']
+    end
+    -- debug(r)
+
+    body = json_decode(r['text'])
+    if last_err() then return end
+
+    if body['result'] ~= 'OK' then
+        return 'JNAP request failed'
+    end
+
+    local now = datetime()
+    for i=1,#body['responses'] do
+        rr = body['responses'][i]['output']
+        if rr['devices'] then
+            for j=1,#rr['devices'] do
+                device = rr['devices'][j]
+                debug(device)
+
+                name=device['friendlyName']
+                macs=device['knownMACAddresses']
+                connections=device['connections']
+
+                for k=1,#macs do
+                    -- this is an historic entry, do not bump last_seen
+                    device_id = db_add('device', {
+                        value=macs[k],
+                        hostname=name,
+                    })
+                    if last_err() then return end
+                end
+
+                for k=1,#connections do
+                    ipaddr = connections[k]['ipAddress']
+                    mac = connections[k]['macAddress']
+
+                    -- this is an active connection
+                    device_id = db_add('device', {
+                        value=mac,
+                        last_seen=now,
+                    })
+                    if last_err() then return end
+
+                    db_add_ttl('network-device', {
+                        network_id=network_id,
+                        device_id=device_id,
+                        ipaddr=ipaddr,
+                        last_seen=now,
+                    }, 180)
+                    if last_err() then return end
+                end
+            end
+        end
+    end
+end

modules/dev/dasoertliche-reverse.lua

@@ -0,0 +1,32 @@
+-- Description: Run phonenumber reverse lookups on dasoertliche.de
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: phonenumbers
+
+function run(arg)
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://www.dasoertliche.de/Controller', {
+        query={
+            form_name='search_inv',
+            ph=arg['value'],
+        }
+    })
+    r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
+
+    x = html_select(r['text'], '#entry_1 a')
+    if not x then
+        -- no results
+        return clear_err()
+    end
+
+    -- strip  
+    m = regex_find('[^\\xa0]+', x['text'])
+
+    if m then
+        db_update('phonenumber', arg, {
+            caller_name=m[1],
+        })
+    end
+end

modules/dev/ddwrt-dhcp.lua

@@ -0,0 +1,88 @@
+-- Description: Export dhcp leases from ddwrt webinterface
+-- Version: 0.2.0
+-- License: GPL-3.0
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    skip_redacted = not getopt('use-redacted')
+
+    router = getopt('router') -- http://192.0.2.1/
+    if not router then
+        return 'router option is missing (http://192.0.2.1/)'
+    end
+    username = getopt('user')
+    password = getopt('password')
+
+    options = {}
+    if username and password then
+        options['basic_auth'] = {username, password}
+    end
+
+    -- request status page
+    session = http_mksession()
+    url = url_join(router, '/Info.live.htm')
+    req = http_request(session, 'GET', url, options)
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then
+        return 'http error: ' .. resp['status']
+    end
+
+    txt = resp['text']
+    debug(txt)
+
+    -- extract leases from response
+    dhcp_section = regex_find('\\{dhcp_leases:: ([^\\}]+)\\}', txt)
+    if last_err() then return end
+    if not dhcp_section then
+        return 'Failed to get dhcp lease section'
+    end
+
+    leases = regex_find_all('\'([^\']+)\',\'([^\']+)\',\'([^\']+)\',\'[^\']+\',\'[^\']+\'', dhcp_section[2])
+    if last_err() then return end
+
+    now = datetime()
+
+    -- add devices to database
+    for i=1, #leases do
+        local hostname = leases[i][2]
+        local ipaddr = leases[i][3]
+        local macaddr = leases[i][4]
+
+        debug({
+            hostname=hostname,
+            ipaddr=ipaddr,
+            macaddr=macaddr,
+        })
+
+        if skip_redacted and macaddr:match('^xx:xx:') then
+            info('Skipping redacted macaddr')
+        else
+            local device = {
+                value=macaddr,
+                last_seen=now,
+            }
+            if hostname ~= '*' then
+                device['hostname'] = hostname
+            end
+
+            local device_id = db_add('device', device)
+
+            db_add_ttl('network-device', {
+                network_id=network_id,
+                device_id=device_id,
+                ipaddr=ipaddr,
+                last_seen=now,
+            }, 120)
+        end
+    end
+end

modules/dev/dns-mx-domains.lua

@@ -0,0 +1,44 @@
+-- Description: Discover mail server from MX records for domain
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function strip_root_dot(name)
+    local m = regex_find("(.+)\\.$", name)
+    if last_err() then return end
+
+    if m == nil then
+        return name
+    else
+        return m[2]
+    end
+end
+
+function run(arg)
+    local records = dns(arg['value'], {
+        record='MX',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+    -- debug(records)
+
+    for i=1, #records do
+        local r = records[i][2]['MX']
+        if r then
+            local mx = strip_root_dot(r[2])
+            local domain = psl_domain_from_dns_name(mx)
+
+            domain_id = db_add('domain', {
+                value=domain,
+            })
+            if last_err() then return end
+
+            db_add('subdomain', {
+                domain_id=domain_id,
+                value=mx,
+            })
+            if last_err() then return end
+        end
+    end
+end

modules/dev/dns-mx-emails.lua

@@ -0,0 +1,54 @@
+-- Description: Discover mail server from MX records for emails
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: emails
+
+function strip_root_dot(name)
+    local m = regex_find("(.+)\\.$", name)
+    if last_err() then return end
+
+    if m == nil then
+        return name
+    else
+        return m[2]
+    end
+end
+
+function run(arg)
+    -- extract domain
+    local domain = arg['value']:match('@(.*)')
+    if doman ~= nil then
+        -- malformed domain
+        return
+    end
+
+    -- mx lookup
+    local records = dns(domain, {
+        record='MX',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+    -- debug(records)
+
+    for i=1, #records do
+        local r = records[i][2]['MX']
+        if r then
+            local mx = strip_root_dot(r[2])
+            domain = psl_domain_from_dns_name(mx)
+
+            domain_id = db_add('domain', {
+                value=domain,
+            })
+            if last_err() then return end
+
+            if domain_id then
+                db_add('subdomain', {
+                    domain_id=domain_id,
+                    value=mx,
+                })
+                if last_err() then return end
+            end
+        end
+    end
+end

modules/dev/dns-ns.lua

@@ -0,0 +1,51 @@
+-- Description: Add a domains NS records to scope
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function strip_root_dot(name)
+    local m = regex_find("(.+)\\.$", name)
+    if last_err() then return end
+
+    if m == nil then
+        return name
+    else
+        return m[2]
+    end
+end
+
+function each(r)
+    local name = strip_root_dot(r)
+    local domain = psl_domain_from_dns_name(name)
+    if last_err() then return end
+
+    -- add domain
+    local domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    -- add subdomain
+    local subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+    if last_err() then return end
+end
+
+function run(arg)
+    local records = dns(arg['value'], {
+        record='NS',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    for i=1, #records do
+        local r = records[i][2]
+        debug(r)
+        each(r['NS'])
+        if last_err() then return end
+    end
+end

modules/dev/dns-ptr.lua

@@ -0,0 +1,30 @@
+-- Description: Run reverse dns lookups
+-- Version: 0.2.0
+-- Source: ipaddrs
+-- License: GPL-3.0
+
+function run(arg)
+    if arg['family'] == '4' then
+        m = regex_find('^(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)$', arg['value'])
+
+        q = m[5] .. '.' .. m[4] .. '.' .. m[3] .. '.' .. m[2] .. '.in-addr.arpa'
+        debug('Resolving: ' .. q)
+
+        records = dns(q, {
+            record='PTR',
+        })
+        if last_err() then return end
+        if records['error'] ~= nil then return end
+        records = records['answers']
+
+        for i=1, #records do
+            r = records[i][2]
+            if r['PTR'] then
+                db_update('ipaddr', arg, {
+                    reverse_dns=r['PTR'],
+                })
+                if last_err() then return end
+            end
+        end
+    end
+end

modules/dev/dns-resolve.lua

@@ -1,25 +1,29 @@
 -- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.1.0
+-- Version: 0.3.0
 -- Source: subdomains
+-- License: GPL-3.0
 
 function run(arg)
     records = dns(arg['value'], 'A')
     if last_err() then return end
 
-    if records['success'] == nil then
-        -- TODO: consider marking the subdomain is inaccessible
+    -- update subdomain
+    resolvable = records['error'] == nil
+    if arg['resolvable'] ~= resolvable then
+        -- TODO: pass arg to function as well
+        db_update('subdomain', arg, {
+            resolvable=resolvable
+        })
+    end
+
+    if not resolvable then
         return
     end
 
-    records = records['success']
+    records = records['answers']
 
-    -- there is a bug in struct -> lua that causes tables to be zero indexed
-    -- this checks if there's something at index 0 but uses index 1 if this is fixed
-    i = 0
-    if records[i] == nil then i = 1 end
-
-    while records[i] ~= nil do
-        r = records[i]
+    for i=1, #records do
+        r = records[i][2]
         if r['A'] ~= nil then
             ipaddr_id = db_add('ipaddr', {
                 family='4',
@@ -31,7 +35,7 @@ function run(arg)
                 subdomain_id=arg['id'],
                 ip_addr_id=ipaddr_id,
             })
+            if last_err() then return end
         end
-        i = i+1
     end
 end

modules/dev/exif.lua

@@ -0,0 +1,12 @@
+-- Description: Extract exif data from images
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    exif = img_exif(arg['value'])
+    if last_err() then return end
+    debug(exif)
+
+    db_update('image', arg, exif)
+end

modules/dev/geoip.lua

@@ -0,0 +1,10 @@
+-- Description: Run a geoip lookup for an ip address
+-- Version: 0.1.0
+-- Source: ipaddrs
+-- License: GPL-3.0
+
+function run(arg)
+    lookup = geoip_lookup(arg['value'])
+    if last_err() then return end
+    db_update('ipaddr', arg, lookup)
+end

modules/dev/git-webroot.lua

@@ -0,0 +1,28 @@
+-- Description: Search for git checkouts in webroot
+-- Version: 0.1.0
+-- Source: urls
+-- License: GPL-3.0
+
+function run(arg)
+    url = url_join(arg['value'], '.git/HEAD')
+
+    session = http_mksession()
+    req = http_request(session, 'GET', url, {})
+    reply = http_send(req)
+    if last_err() then return end
+
+    if reply['status'] ~= 200 then
+        return
+    end
+
+    if not regex_find('^ref: ', reply['text']) then
+        return
+    end
+
+    db_add('url', {
+        subdomain_id=arg['subdomain_id'],
+        value=url,
+        status=reply['status'],
+        body=reply['text'],
+    })
+end

modules/dev/github.lua

@@ -0,0 +1,107 @@
+-- Description: Collect data from github profiles
+-- Version: 0.2.0
+-- Source: accounts:github.com
+-- License: GPL-3.0
+
+function api_get(url)
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+    -- TODO: set_error(?)
+    if resp['status'] == 403 then return 'ratelimit exceeded' end
+    if resp['status'] ~= 200 then return 'invalid status code' end
+
+    local data = json_decode(resp['text'])
+    if last_err() then return end
+    return data
+end
+
+function import_gpg(url)
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+
+    local key = pgp_pubkey_armored(resp['text'])
+    if not key['uids'] then return end
+
+    for i=1, #key['uids'] do
+        local k = key['uids'][i]
+        debug(k)
+        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", k)
+        if m then
+            db_add('email', {
+                value=m[3],
+                displayname=m[2],
+            })
+        end
+    end
+end
+
+function scan4email(username)
+    local url = 'https://api.github.com/users/' .. username .. '/repos'
+    local repos = api_get(url)
+    if last_err() then return end
+
+    -- XXX: 'https://api.github.com/users/' .. username .. '/events/public?page=0&per_page=100' is faster but less accurate
+
+    for i=1, #repos do
+        local repo = repos[i]
+        debug(repo)
+        local commits = api_get(repo['url'] .. '/commits')
+        if last_err() then return end
+
+        for j=1, #commits do
+            local commit = commits[j]
+            debug(commit)
+
+            if commit['author'] and commit['author']['login'] == username then
+                local name = commit['commit']['author']['name']
+                local email = commit['commit']['author']['email']
+                db_add('email', {
+                    value=email,
+                    displayname=name,
+                })
+                return email
+            end
+
+            if commit['committer'] and commit['committer']['login'] == username then
+                local name = commit['commit']['committer']['name']
+                local email = commit['commit']['committer']['email']
+                db_add('email', {
+                    value=email,
+                    displayname=name,
+                })
+                return email
+            end
+        end
+    end
+end
+
+function run(arg)
+    session = http_mksession()
+    local url = 'https://api.github.com/users/' .. arg['username']
+
+    local data = api_get(url)
+    if last_err() then return end
+    debug(data)
+
+    -- company = data['company']
+    -- location = data['location']
+    -- homepage = data['blog']
+
+    url = 'https://github.com/' .. arg['username'] .. '.gpg'
+    import_gpg(url)
+    if last_err() then return end
+
+    local email = data['email']
+    if not email and not arg['email'] then
+        email = scan4email(arg['username'])
+        if last_err() then return end
+    end
+
+    db_update('account', arg, {
+        url=data['html_url'],
+        displayname=data['name'],
+        email=email,
+    })
+end