Release v0.21.1

Update dependencies to fix aarch64 on musl

.dockerignore

@@ -1,6 +1,10 @@
 target
 Dockerfile
 .dockerignore
+docker-compose.yml
+docker
+docs
+ci
 .git
 .gitignore
 *.sw[op]

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [kpcyrd]

.github/workflows/docker.yml

@@ -0,0 +1,30 @@
+name: Docker
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+        - name: sn0int
+          file: Dockerfile
+        - name: registry
+          file: sn0int-registry/Dockerfile
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Build the Docker image
+      run: docker build -t ${{ matrix.build.name }} -f ${{ matrix.build.file }} .
+    - name: Test the Docker image
+      run: docker run --rm ${{ matrix.build.name }} --help
+
+    - name: Show Docker images
+      run: docker images

.github/workflows/rust.yml

@@ -0,0 +1,69 @@
+name: Rust
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 9 * * 1'
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install dependencies (apt)
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get install libsqlite3-dev libseccomp-dev libsodium-dev
+    - name: Install dependencies (brew)
+      if: matrix.os == 'macos-latest'
+      run: brew install pkg-config libsodium
+
+    - name: Build (sn0int)
+      run: cargo build --verbose
+    - name: Build (common)
+      run: cd sn0int-common && cargo build --verbose
+    - name: Build (std)
+      run: cd sn0int-std && cargo build --verbose
+    - name: Build (examples)
+      run: cargo build --verbose --examples
+
+    - name: Run tests (sn0int)
+      run: cargo test --verbose
+    - name: Run tests (sn0int, --ignored)
+      run: cargo test --verbose -- --ignored
+    - name: Run tests (common)
+      run: cd sn0int-common && cargo test --verbose
+    - name: Run tests (common, --ignored)
+      run: cd sn0int-common && cargo test --verbose -- --ignored
+    - name: Run tests (std)
+      run: cd sn0int-std && cargo test --verbose
+    - name: Run tests (std, --ignored)
+      run: cd sn0int-std && cargo test --verbose -- --ignored
+
+  notify:
+    # forks shouldn't notify
+    if: github.repository == 'kpcyrd/sn0int'
+    runs-on: ubuntu-latest
+    needs:
+    - build
+
+    steps:
+    - name: irc notify
+      uses: rectalogic/notify-irc@v1
+      with:
+        server: irc.hackint.org
+        channel: "#sn0int"
+        nickname: github-ci
+        message: '${{ github.repository }}#${{ github.run_id }}(${{ github.event_name }}): ${{ github.ref }}: tests completed: ${{ needs.build.result }} (https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})'

.travis.yml

@@ -1,57 +0,0 @@
-language: rust
-
-cache: cargo
-
-matrix:
-  include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker-registry
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  #- os: windows
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE="windows test"
-  - os: windows
-    rust: stable
-    env:
-    - BUILD_MODE="windows common"
-
-before_install:
-- ci/setup.sh "$TRAVIS_OS_NAME"
-script:
-- df -h
-- ci/run.sh $BUILD_MODE
-- df -h
-
-notifications:
-  irc:
-    channels:
-    - "ircs://irc.hackint.org:6697/#sn0int"
-    #on_success: change # default: always
-    #on_failure: always # default: always
-    use_notice: true

CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# How to contribute
+
+To contribute to sn0int, clone the repository and make sure both the build and
+tests pass for you:
+
+    git clone https://github.com/kpcyrd/sn0int.git
+    cd sn0int
+    # build the project
+    cargo build
+    # run regular tests
+    cargo test
+    # run tests depending on the network
+    # these might fail if a service is down
+    cargo test -- --ignored
+
+The project is loosely structured into a few folders:
+
+- `src/models/` - database models
+- `src/runtime/` - the stdlib that's exposed to lua
+- `src/engine/` - code related to lua
+- `src/sandbox/` - code related to sandboxing
+- `src/cmd/` - cli commands
+- `src/` - misc modules
+
+After you're done, make sure the build completes without any warnings and both
+tests pass successfully:
+
+    cargo test
+    cargo test -- --ignored
+
+If you want to introduce a new feature feel free to open an issue first to make
+sure your feature is a good fit for the project before implementing it.

Cargo.toml

@@ -1,69 +1,106 @@
 [package]
 name = "sn0int"
-version = "0.5.2"
-description = "OSINT framework and package manager"
+version = "0.21.1"
+description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
-
-[badges]
-travis-ci = { repository = "kpcyrd/sn0int" }
+edition = "2018"
 
 [workspace]
-members = ["sn0int-registry/sn0int-common",
-           "sn0int-registry"]
+members = ["sn0int-common",
+           "sn0int-registry",
+           "sn0int-std"]
+
+[package.metadata.deb]
+extended-description = """\
+sn0int (pronounced /snoɪnt/) is a semi-automatic OSINT framework and package
+manager. It was built for IT security professionals and bug hunters to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations."""
+section = "utils"
+priority = "optional"
+depends = "$auto, publicsuffix"
+build-depends = "pkg-config, libsqlite3-dev, libseccomp-dev, libsodium-dev, publicsuffix"
+assets = [
+    ["target/release/sn0int", "usr/bin/", "755"],
+]
+
+[features]
+sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.3.0", path="sn0int-registry/sn0int-common" }
-rustyline = "2"
+sn0int-common = { version="0.12.0", path="sn0int-common" }
+sn0int-std = { version="=0.21.0", path="sn0int-std" }
+rustyline = "8.0"
 log = "0.4"
-env_logger = "0.6"
+env_logger = "0.8"
 hlua-badtouch = "0.4"
-structopt = "0.2"
+structopt = "0.3"
 failure = "0.1"
-rand = "0.6"
-colored = "1.6"
+rand = "0.8"
+colored = "2"
 lazy_static = "1.0"
 shellwords = "1.0"
-publicsuffix = { version="1.5", default-features=false }
-diesel = { version = "1.0.0", features = ["sqlite"] }
+diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-dirs = "1.0"
-url = "1.7"
-chrootable-https = "0.3.4"
-trust-dns-proto = "0.5.0"
-base64 = "0.10"
-kuchiki = "0.7.2"
-serde_urlencoded = "0.5"
-serde = "1.0"
-serde_derive = "1.0"
+libsqlite3-sys = { version = "0.18.0", features = ["bundled-windows"] }
+chrono = { version = "0.4", features = ["serde"] }
+dirs-next = "2.0"
+url = "2.0"
+percent-encoding = "2.1"
+#chrootable-https = { path = "../chrootable-https" }
+chrootable-https = "0.16"
+base64 = "0.13"
+data-encoding = "2.1.2"
+serde = { version = "1.0", features = ["derive"] }
+serde_urlencoded = "0.7"
 serde_json = "1.0"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.5"
 ctrlc = "3.1"
-opener = "0.3.0"
-separator = "0.3.1"
+opener = "0.4"
+separator = "0.4"
 maplit = "1.0.1"
-sloppy-rfc4880 = "0.1.2"
+sloppy-rfc4880 = "0.2"
 regex = "1.0"
-toml = "0.4"
-maxminddb = "0.10.0"
-tar = "0.4.17"
-libflate = "0.1.14"
+toml = "0.5"
 threadpool = "1.7"
-x509-parser = "0.4.0"
-der-parser = "1.1.0"
-nom = "4.1.1"
+atty = "0.2"
+semver = "0.11"
+bytes = "0.4"
+bytesize = "1.0"
+ipnetwork = "0.18"
+strum = "0.20"
+strum_macros = "0.20"
+embedded-triple = "0.1.0"
+humansize = "1.1.0"
+
+digest = "0.9"
+md-5 = "0.9"
+sha-1 = "0.9"
+sha2 = "0.9"
+sha3 = "0.9"
+hmac = "0.11"
+
+walkdir = "2.2"
+nude = "0.3"
+glob = "0.3.0"
+os-version = "0.1.1"
 
 [target.'cfg(target_os="linux")'.dependencies]
-caps = "0.3"
-syscallz = "0.7"
-nix = "0.11"
+caps = "0.5"
+#syscallz = { path="../syscallz-rs" }
+syscallz = "0.15"
+nix = "0.20"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
-pledge = "0.3.1"
-unveil = "0.2.0"
+pledge = "0.4"
+unveil = "0.3"
 
 [dev-dependencies]
-boxxy = "0.8"
+#boxxy = { path = "../boxxy-rs" }
+boxxy = "0.11"
+tempfile = "3.0"

Dockerfile

@@ -1,13 +1,13 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

ISSUE_TEMPLATE.md

@@ -0,0 +1,29 @@
+<!--
+Hello!
+
+If you want to report a bug we added some common questions below that help us analyse your issue faster.
+
+All of these are optional so feel free to remove anything that doesn't apply.
+-->
+
+please describe your issue here
+
+---
+
+## Versions
+
+- **rustc --version:**
+- **cargo --version:**
+- **sn0int --version:**
+- **uname -a:**
+
+## Environment
+
+- **Operating System/Distro:**
+- **Installed from (source/apt/pacman/brew/docker):**
+
+<!--
+Thank you!
+
+We'll try to respond as quickly as possible.
+-->

Makefile

@@ -1,16 +1,32 @@
 check:
-	(cd sn0int-registry/sn0int-common; cargo check)
+	(cd sn0int-common; cargo check)
 	(cd sn0int-registry; cargo check)
+	(cd sn0int-std; cargo check)
 	cargo check
 
 force-check:
-	(cd sn0int-registry/sn0int-common; touch src/lib.rs; cargo check)
+	(cd sn0int-common; touch src/lib.rs; cargo check)
 	(cd sn0int-registry; touch src/main.rs; cargo check)
+	(cd sn0int-std; touch src/lib.rs; cargo check)
 	touch src/lib.rs
 	cargo check
 
 test:
-	(cd sn0int-registry/sn0int-common; cargo test)
+	(cd sn0int-common; cargo test)
 	(cd sn0int-registry; cargo test)
-	cargo test
-	cargo test -- --ignored
+	(cd sn0int-std; cargo test)
+	(cd sn0int-std; cargo test -- --ignored)
+	cargo test --lib
+	cargo test --lib -- --ignored
+
+update:
+	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt
+	get-iab -v -u http://standards-oui.ieee.org/iab/iab.txt -f data/ieee-iab.txt
+	rm -f data/ieee-*.txt.bak
+
+docs:
+	$(MAKE) -C docs html
+	contrib/html-toc2md.pl README.md docs/_build/html/index.html > README2.md
+	mv README2.md README.md
+
+.PHONY: check force-check test update docs

README.md

@@ -1,44 +1,324 @@
-# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs]
+# sn0int [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
-[travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
-[travis]:       https://travis-ci.org/kpcyrd/sn0int
 [crates-img]:   https://img.shields.io/crates/v/sn0int.svg
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
 [docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
+[irc-img]:      https://img.shields.io/badge/hackint-%23sn0int-blue.svg
+[irc]:          https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int
+[twitter-img]:  https://img.shields.io/badge/twitter-@sn0int-blue.svg
+[twitter]:      https://twitter.com/sn0int
+[mastodon-img]: https://img.shields.io/badge/mastodon-chaos.social-blue.svg
+[mastodon]:     https://chaos.social/@sn0int
+[registry-img]: https://img.shields.io/website/https/sn0int.com.svg?label=registry
+[registry]:     https://sn0int.com/
 
-sn0int is an OSINT framework and package manager. It was built for IT security
-professionals and bug hunters to gather intelligence about a given target or
-about yourself. sn0int is enumerating attack surface by semi-automatically
-processing public information and mapping the results in a unified format for
-followup investigations.
+sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
+package manager. It was built for IT security professionals and bug hunters to
+gather intelligence about a given target or about yourself. sn0int is
+enumerating attack surface by semi-automatically processing public information
+and mapping the results in a unified format for followup investigations.
+
+[ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
 
 Among other things, sn0int is currently able to:
 
-- [X] Harvest subdomains from certificate transparency logs
-- [X] Harvest subdomains from various passive dns logs
-- [X] Sift through subdomain results for publicly accessible websites
-- [X] Harvest emails from pgp keyservers
-- [X] Enrich ip addresses with ASN and geoip info
-- [X] Harvest subdomains from the wayback machine
+- Harvest subdomains from certificate transparency logs and passive dns
+- Enrich ip addresses with asn and geoip info
+- Harvest emails from pgp keyservers and whois
+- Discover compromised logins in breaches
+- Find somebody's profiles across the internet
+- Enumerate local networks with unique techniques like passive arp
+- Gather information about phonenumbers
+- Attempt to bypass cloudflare with shodan
+- Harvest data and images from instagram profiles
+- Scan images for nudity
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
-and is fully opensource.  None of the investigations listed above are hardcoded
+and is fully opensource. None of the investigations listed above are hardcoded
 in the source, instead those are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
 you to ship updates for your modules on your own since you don't need to send a
 pull request.
 
-Join us on IRC: <ircs://irc.hackint.org/#sn0int>
+For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
+
+[![asciicast](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB.svg)](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB)
 
 ## Installation
 
-- Archlinux: `pacman -S sn0int`
-- Alpine: `apk add --no-cache sqlite-dev libseccomp-dev cargo` + build from source
-- Debian: `apt install libsqlite3-dev libseccomp-dev` + build from source
-- OpenBSD: `pkg_add sqlite3` + build from source
-- OSX: `brew install sqlite3` + build from source
+<a href="https://repology.org/project/sn0int/versions"><img align="right" src="https://repology.org/badge/vertical-allrepos/sn0int.svg" alt="Packaging status"></a>
+
+Archlinux
+
+    pacman -S sn0int
+
+Mac OSX
+
+    brew install sn0int
+
+Debian/Ubuntu/Kali
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+    apt install debian-keyring
+    gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | apt-key add -
+    apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    echo deb http://apt.vulns.sexy stable main > /etc/apt/sources.list.d/apt-vulns-sexy.list
+    apt update
+    apt install sn0int
+
+Docker
+
+    docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
+
+Alpine
+
+    apk add sn0int
+
+OpenBSD
+
+    pkg_add sn0int
+
+Gentoo
+
+    layman -a pentoo
+    emerge --ask net-analyzer/sn0int
+
+NixOS
+
+    nix-env -i sn0int
+
+For everything else please have a look at the [detailed list][1].
+
+[1]: https://sn0int.readthedocs.io/en/latest/install.html
+
+## Getting started
+
+- [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
+  - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
+  - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
+  - [Debian &gt;= bullseye, Ubuntu &gt;= 20.04, Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-bullseye-ubuntu-20-04-kali)
+  - [Debian &lt;= buster, Ubuntu &lt;= 19.10](https://sn0int.readthedocs.io/en/latest/install.html#debian-buster-ubuntu-19-10)
+  - [Fedora/CentOS/Redhat](https://sn0int.readthedocs.io/en/latest/install.html#fedora-centos-redhat)
+  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
+  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
+  - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
+  - [NixOS](https://sn0int.readthedocs.io/en/latest/install.html#nixos)
+  - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
+- [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
+  - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
+    - [Archlinux](https://sn0int.readthedocs.io/en/latest/build.html#archlinux)
+    - [Mac OSX](https://sn0int.readthedocs.io/en/latest/build.html#mac-osx)
+    - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/build.html#debian-ubuntu-kali)
+    - [Alpine](https://sn0int.readthedocs.io/en/latest/build.html#alpine)
+    - [OpenBSD](https://sn0int.readthedocs.io/en/latest/build.html#openbsd)
+    - [Gentoo](https://sn0int.readthedocs.io/en/latest/build.html#gentoo)
+    - [Windows](https://sn0int.readthedocs.io/en/latest/build.html#windows)
+  - [Building](https://sn0int.readthedocs.io/en/latest/build.html#building)
+- [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
+  - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
+  - [Adding something to scope](https://sn0int.readthedocs.io/en/latest/usage.html#adding-something-to-scope)
+  - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
+  - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
+  - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
+- [Autonoscope](https://sn0int.readthedocs.io/en/latest/autonoscope.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/autonoscope.html#domains)
+  - [IPs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#ips)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#urls)
+- [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
+  - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
+  - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Publish your repo](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-repo)
+  - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
+- [Database](https://sn0int.readthedocs.io/en/latest/database.html)
+  - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/database.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/database.html#db-activity)
+  - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
+  - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
+- [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/structs.html#domains)
+  - [Subdomains](https://sn0int.readthedocs.io/en/latest/structs.html#subdomains)
+  - [IpAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#ipaddrs)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/structs.html#urls)
+  - [Emails](https://sn0int.readthedocs.io/en/latest/structs.html#emails)
+  - [Phonenumbers](https://sn0int.readthedocs.io/en/latest/structs.html#phonenumbers)
+  - [Devices](https://sn0int.readthedocs.io/en/latest/structs.html#devices)
+  - [Networks](https://sn0int.readthedocs.io/en/latest/structs.html#networks)
+  - [Accounts](https://sn0int.readthedocs.io/en/latest/structs.html#accounts)
+  - [Breaches](https://sn0int.readthedocs.io/en/latest/structs.html#breaches)
+  - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
+  - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
+  - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
+  - [CryptoAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#cryptoaddrs)
+  - [Activity](https://sn0int.readthedocs.io/en/latest/structs.html#activity)
+  - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
+    - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
+    - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
+    - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Activity](https://sn0int.readthedocs.io/en/latest/activity.html)
+  - [Anatomy of an event](https://sn0int.readthedocs.io/en/latest/activity.html#anatomy-of-an-event)
+  - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
+  - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
+  - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
+- [Notifications](https://sn0int.readthedocs.io/en/latest/notifications.html)
+  - [Receiving notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#receiving-notifications)
+    - [Telegram](https://sn0int.readthedocs.io/en/latest/notifications.html#telegram)
+    - [Pushover](https://sn0int.readthedocs.io/en/latest/notifications.html#pushover)
+    - [Discord](https://sn0int.readthedocs.io/en/latest/notifications.html#discord)
+    - [Signal](https://sn0int.readthedocs.io/en/latest/notifications.html#signal)
+    - [Writing your own module](https://sn0int.readthedocs.io/en/latest/notifications.html#writing-your-own-module)
+  - [Setting up notification rules](https://sn0int.readthedocs.io/en/latest/notifications.html#setting-up-notification-rules)
+  - [Testing notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#testing-notifications)
+  - [Running sn0int automatically](https://sn0int.readthedocs.io/en/latest/notifications.html#running-sn0int-automatically)
+    - [Monitors](https://sn0int.readthedocs.io/en/latest/notifications.html#monitors)
+    - [Timers](https://sn0int.readthedocs.io/en/latest/notifications.html#timers)
+- [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
+  - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
+  - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
+  - [Using access keys as source argument](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-as-source-argument)
+- [Configuration](https://sn0int.readthedocs.io/en/latest/config.html)
+  - [\[core\]](https://sn0int.readthedocs.io/en/latest/config.html#core)
+  - [\[namespaces\]](https://sn0int.readthedocs.io/en/latest/config.html#namespaces)
+  - [\[network\]](https://sn0int.readthedocs.io/en/latest/config.html#network)
+- [Sandbox](https://sn0int.readthedocs.io/en/latest/sandbox.html)
+  - [Linux](https://sn0int.readthedocs.io/en/latest/sandbox.html#linux)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/sandbox.html#openbsd)
+  - [IPC Protocol](https://sn0int.readthedocs.io/en/latest/sandbox.html#ipc-protocol)
+  - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
+  - [Diagnosing a sandbox failure](https://sn0int.readthedocs.io/en/latest/sandbox.html#diagnosing-a-sandbox-failure)
+- [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
+  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
+  - [base64_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-decode)
+  - [base64_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-encode)
+  - [base64_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-decode)
+  - [base64_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-encode)
+  - [base32_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-decode)
+  - [base32_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-encode)
+  - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
+  - [create_blob](https://sn0int.readthedocs.io/en/latest/reference.html#create-blob)
+  - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
+  - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/reference.html#db-activity)
+  - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
+  - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
+  - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
+  - [error](https://sn0int.readthedocs.io/en/latest/reference.html#error)
+  - [geoip_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#geoip-lookup)
+  - [hex](https://sn0int.readthedocs.io/en/latest/reference.html#hex)
+  - [hmac_md5](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-md5)
+  - [hmac_sha1](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha1)
+  - [hmac_sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-256)
+  - [hmac_sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-512)
+  - [hmac_sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-256)
+  - [hmac_sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-512)
+  - [html_select](https://sn0int.readthedocs.io/en/latest/reference.html#html-select)
+  - [html_select_list](https://sn0int.readthedocs.io/en/latest/reference.html#html-select-list)
+  - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
+  - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
+  - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [http_fetch](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch)
+  - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
+  - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
+  - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
+  - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)
+  - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
+  - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
+  - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [key_trunc_pad](https://sn0int.readthedocs.io/en/latest/reference.html#key-trunc-pad)
+  - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
+  - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
+  - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [mqtt_connect](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-connect)
+  - [mqtt_subscribe](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-subscribe)
+  - [mqtt_recv](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-recv)
+  - [mqtt_ping](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-ping)
+  - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
+  - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
+  - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
+  - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [ratelimit_throttle](https://sn0int.readthedocs.io/en/latest/reference.html#ratelimit-throttle)
+  - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
+  - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
+  - [set_err](https://sn0int.readthedocs.io/en/latest/reference.html#set-err)
+  - [sha1](https://sn0int.readthedocs.io/en/latest/reference.html#sha1)
+  - [sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-256)
+  - [sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-512)
+  - [sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-256)
+  - [sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-512)
+  - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [sn0int_time](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time)
+  - [sn0int_time_from](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time-from)
+  - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
+  - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
+  - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
+  - [sock_options](https://sn0int.readthedocs.io/en/latest/reference.html#sock-options)
+  - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
+  - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
+  - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
+  - [sock_recvline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline)
+  - [sock_recvall](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvall)
+  - [sock_recvline_contains](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline-contains)
+  - [sock_recvline_regex](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline-regex)
+  - [sock_recvn](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvn)
+  - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
+  - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
+  - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [sodium_secretbox_open](https://sn0int.readthedocs.io/en/latest/reference.html#sodium-secretbox-open)
+  - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
+  - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [str_find](https://sn0int.readthedocs.io/en/latest/reference.html#str-find)
+  - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
+  - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
+  - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [strval](https://sn0int.readthedocs.io/en/latest/reference.html#strval)
+  - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
+  - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
+  - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
+  - [url_escape](https://sn0int.readthedocs.io/en/latest/reference.html#url-escape)
+  - [url_join](https://sn0int.readthedocs.io/en/latest/reference.html#url-join)
+  - [url_parse](https://sn0int.readthedocs.io/en/latest/reference.html#url-parse)
+  - [url_unescape](https://sn0int.readthedocs.io/en/latest/reference.html#url-unescape)
+  - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
+  - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
+  - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
+  - [ws_connect](https://sn0int.readthedocs.io/en/latest/reference.html#ws-connect)
+  - [ws_options](https://sn0int.readthedocs.io/en/latest/reference.html#ws-options)
+  - [ws_recv_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-text)
+  - [ws_recv_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-binary)
+  - [ws_recv_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-json)
+  - [ws_send_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-text)
+  - [ws_send_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-binary)
+  - [ws_send_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-json)
+  - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
+  - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
+  - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)
+
+## Rationale
+
+This tool was written for companies to help them understand their attack
+surface from a blackbox point of view. It's often difficult to understand that
+something is easier to discover than some people assume, putting them at risk
+of false security.
+
+It's also designed to be useful for red team assessments and bug bounties,
+which also help companies to identify weaknesses that could result in a
+compromise.
+
+Some functionality was written to do the same thing for individuals to raise
+awareness about personal attack surface, privacy and how much data is publicly
+available. These issues are often out of scope in bug bounties and sometimes by
+design. We believe that blaming the user is the wrong approach and these issues
+should be addressed at the root cause by the people designing those systems.
 
 ## License
 

ci/boxxy_stage1.txt

@@ -1,3 +1,5 @@
 stage1
 ls
+echo checkpoint1
+id
 echo sandbox fail

ci/integration.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+import subprocess
+from subprocess import DEVNULL, PIPE
+import tempfile
+import json
+import sys
+
+
+def _sn0int(tempdir, binary, args, piped_stdout=False):
+    return subprocess.Popen(
+        ['/usr/bin/env', 'HOME='+tempdir, binary] + args,
+        stdin=PIPE,
+        stdout=PIPE if piped_stdout else None,
+    )
+
+
+def sn0int(tempdir, binary, cmds):
+    p = _sn0int(tempdir, binary, [])
+    for cmd in cmds:
+        p.stdin.write((cmd + '\n').encode('utf-8'))
+    p.communicate()
+    if p.returncode != 0:
+        raise Exception('process failed')
+
+
+def sn0int_select(tempdir, binary, query):
+    p = _sn0int(tempdir, binary, ['select', '--json'] + query, piped_stdout=True)
+    stdout, _ = p.communicate()
+    lines = filter(None, stdout.decode('utf-8').split('\n'))
+    return [json.loads(x) for x in lines]
+
+
+def main(tempdir, binary):
+    print('[*] setting up workspace')
+    sn0int(tempdir, binary, [])
+
+    print('[*] adding domain')
+    sn0int(tempdir, binary, [
+        'add domain',
+        'example.com',
+        'select domains',
+    ])
+
+    print('[*] testing db for domain')
+    domains = sn0int_select(tempdir, binary, ['domains'])
+    assert domains == [{'id': 1, 'value': 'example.com', 'unscoped': False}]
+
+    print('[*] installing modules')
+    sn0int(tempdir, binary, [
+        'pkg install kpcyrd/ctlogs',
+        'pkg install kpcyrd/dns-resolve',
+        'pkg install kpcyrd/url-scan',
+        'pkg install kpcyrd/geoip',
+    ])
+
+    print('[*] running ctlogs')
+    sn0int(tempdir, binary, [
+        'use ctlogs',
+        'run',
+        'select subdomains',
+    ])
+
+    print('[*] testing db for subdomains')
+    subdomains = sn0int_select(tempdir, binary, ['subdomains'])
+    assert {x['value'] for x in subdomains} == {
+        'example.com',
+        'www.example.com',
+        'm.example.com',
+        'dev.example.com',
+        'products.example.com',
+        'support.example.com',
+    }
+
+    print('[*] running dns-resolve')
+    sn0int(tempdir, binary, [
+        'use dns-resolve',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs')
+    ipaddrs = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert len(ipaddrs) >= 1
+
+    print('[*] running url-scan')
+    sn0int(tempdir, binary, [
+        'use url-scan',
+        'run',
+        'select urls',
+    ])
+
+    print('[*] testing db for urls')
+    urls = sn0int_select(tempdir, binary, ['urls'])
+    assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
+        ('http://www.example.com/', 200),
+        ('https://www.example.com/', 200),
+    }
+
+    print('[*] running geoip')
+    sn0int(tempdir, binary, [
+        'use geoip',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs again')
+    ipaddrs2 = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert ipaddrs != ipaddrs2
+
+    print('')
+    print('\t###########')
+    print('\t# SUCCESS #')
+    print('\t###########')
+    print('')
+
+
+if __name__ == '__main__':
+    try:
+        binary = sys.argv[1]
+    except IndexError:
+        print('Usage: %s target/release/sn0int' % sys.argv[0])
+    else:
+        with tempfile.TemporaryDirectory(prefix='sn0int-') as tempdir:
+            main(tempdir, binary)

ci/run.sh

@@ -1,41 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    build)
-        cargo build --verbose
-        cargo build --verbose --examples
-        ;;
-    test)
-        ci/run.sh build
-        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
-             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind-dl -- -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind-dl -- -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
-        cargo test --verbose
-        cargo test --verbose -- --ignored
-        ;;
-    common)
-        cd sn0int-registry/sn0int-common
-        cargo test --verbose
-        ;;
-    windows)
-        export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
-        ci/run.sh "$2"
-        ;;
-    boxxy)
-        cargo build --verbose --examples
-        if cat ci/boxxy_stage1.txt | RUST_LOG=boxxy cargo run --example boxxy; then
-            echo "SANDOX ERROR: should've crashed"
-            exit 1
-        fi
-        ;;
-    docker)
-        docker build -t sn0int .
-        docker images
-        docker run --rm sn0int --help
-        ;;
-    docker-registry)
-        docker build -t sn0int-registry sn0int-registry/
-        docker images
-        ;;
-esac

ci/setup.sh

@@ -1,16 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    linux)
-        sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev
-        ;;
-    osx)
-        brew install sqlite3
-        ;;
-    windows)
-        curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-        7z e sqlite3.zip -y
-        "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
-        ;;
-esac

contrib/docker/Dockerfile.alpine

@@ -0,0 +1,15 @@
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
+WORKDIR /usr/src/sn0int
+COPY . .
+RUN cargo build --release --verbose
+RUN strip target/release/sn0int
+
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
+COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+VOLUME ["/data", "/cache"]
+ENV XDG_DATA_HOME=/data \
+    XDG_CACHE_HOME=/cache
+ENTRYPOINT ["sn0int"]

contrib/docker/Dockerfile.debian

@@ -0,0 +1,16 @@
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /usr/src/sn0int
+COPY . .
+RUN cargo build --release --verbose
+RUN strip target/release/sn0int
+
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+VOLUME ["/data", "/cache"]
+ENV XDG_DATA_HOME=/data \
+    XDG_CACHE_HOME=/cache
+ENTRYPOINT ["sn0int"]

contrib/html-toc2md.pl

@@ -0,0 +1,39 @@
+#!/usr/bin/env perl
+use strict; use warnings;
+
+my ($readme, $toc) = @ARGV;
+defined $readme or die 'missing readme path';
+defined $toc or die 'missing toc path';
+
+open(my $r, "<$readme") or die 'failed to open readme';
+open(my $t, "<$toc") or die 'failed to open toc';
+
+my $re = qr/^\s*- \[.+\]\(https:\/\/sn0int.readthedocs.io\/en\/.+\)$/;
+
+# pass through start of readme
+while (<$r>) {
+    last if ($_ =~ $re);
+    print $_;
+}
+
+# skip toc
+while (<$r>) {
+    last unless ($_ =~ $re);
+}
+
+# generate new toc
+while (my $line = <$t>) {
+    if ($line =~ /toctree-l(\d).*href="([^"]+)">(.+)<\/a/) {
+        my $space = int($1)-1;
+        my $section = $2;
+        my $label = $3;
+        $label =~ s/([\[\]])/\\$1/g;
+        print " " x ($space*2), "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
+    }
+}
+print;
+
+# pass through end of readme
+while (<$r>) {
+    print $_;
+}

docs/activity.rst

@@ -0,0 +1,166 @@
+Activity
+========
+
+So far we've learned about regular `structs <structs.html>`_, but activity is
+special.
+
+Activity is an event tied to a specific time and topic and has a small amount
+of data piggybacked to it.
+
+Anatomy of an event
+-------------------
+
+``topic``
+    This is some freestyle text used to group events to a specific topic. This
+    must not conflict with other modules unless there's a very good reason.
+
+    The topic should look like ``kpcyrd/example:something``, with ``something``
+    being a meaningful unique identifier for whatever is generating these
+    events, like a mac address or an account name/id.
+
+    The rules around this might become stricter in the future.
+``time``
+    The most important part of the event: The time and date it happened.
+``initial``
+    This value can not be set but might be present in sn0int output. See `Querying events`_.
+``uniq`` (optional)
+    This is an optional feature to deduplicate events. Assuming you're
+    importing posts by an account, you wouldn't want to store a new event for
+    each post you already imported. If you set this field to the technical post
+    id then sn0int would skip the event if it already has an event with the
+    same ``topic`` and ``uniq`` combination to avoid inserting duplicates.
+``latitude`` (optional)
+    Latitude - if you can tie the event to a specific location.
+``longitude`` (optional)
+    Longitude - if you can tie the event to a specific location.
+``radius`` (optional)
+    The location radius in meters. If the position you got has a precision of
+    100 meters set this value to ``100``.
+``content``
+    Arbitrary data that you want to attach to the event. This doesn't need to
+    be a string and can be an arbitrary object that is then stored as json
+    string.
+
+Logging events
+--------------
+
+An ``activity`` event can be logged with ``db_activity``:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Logging an event that has a location attached could look like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        latitude=40.726662,
+        longitude=-74.036677,
+        radius=50,
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Making sure an event is not logged twice can be done with ``uniq``:
+
+.. code-block:: lua
+
+    -- create the first event
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this does nothing because we already have an event with this topic+uniq combination
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this creates a new event because uniq is different
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+    -- this also creates a new event because topic is different
+    db_activity({
+        topic='harness/activity-ping:something-else',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+Querying events
+---------------
+
+There is a commandline interface that can be used to query all events we've
+logged. To get everything (sorted by time)::
+
+    sn0int activity
+
+To limit the output to a specific topic::
+
+    sn0int activity -t harness/activity-ping:dummy
+
+To limit it to a specific time frame::
+
+    # everything since
+    sn0int activity --since 2020-01-13T04:20:00
+    # everything until
+    sn0int activity --until 2020-01-13T04:20:00
+    # both
+    sn0int activity --since yesterday --until today
+
+When using ``--since`` you might also want to know the previous state and use
+it as an initial value. Consider this example::
+
+    2020-01-13 14:30:00 # user goes offline
+    2020-01-13 23:59:00 # user goes online
+    2020-01-14 09:30:00 # user goes idle
+    2020-01-14 14:20:00 # user goes offline
+
+If we're running a query like ``sn0int activity --since 2020-01-14T00:00:00``
+the program consuming the output wouldn't know that the user is initially
+online because we're only getting this data::
+
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+We can tweak this with ``sn0int activity --initial --since
+2020-01-14T00:00:00`` to include one more event that we only use to populate
+the intial state::
+
+    {"id":7,"initial":true,"topic":"foo/bar:asdf","time":"2020-01-13T23:59:00","content":{"state":"online"}}
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+Visualization
+-------------
+
+There is no visualization built in, there may be external frontends for this in
+the future. You're very welcome to write one!

docs/autonoscope.rst

@@ -0,0 +1,81 @@
+Autonoscope
+===========
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+-------
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+---
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+----
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

docs/build.rst

@@ -0,0 +1,76 @@
+Build from source
+=================
+
+It's generally recommended to `install a package <install.html>`_ if available.
+This section is about building the binary from git.
+
+Install dependencies
+--------------------
+
+You need a recent rust compiler. It's usually recommended to install a rust
+compiler with `rustup <https://rustup.rs/>`_, but if you're system ships the
+most recent compiler in a package that works too. Note that some systems aren't
+fully supported by rustup (like OpenBSD and alpine) and you need to install
+rust from a package in that case.
+
+Archlinux
+~~~~~~~~~
+
+.. code-block:: bash
+
+    $ pacman -S geoip2-database libseccomp libsodium publicsuffix-list sqlite
+
+Mac OSX
+~~~~~~~
+
+.. code-block:: bash
+
+    $ brew install libsodium
+
+Debian/Ubuntu/Kali
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix pkg-config
+
+.. warning::
+   On a debian based system make sure you've installed rust with rustup.
+
+Alpine
+~~~~~~
+
+.. code-block:: bash
+
+    $ apk add sqlite-dev libseccomp-dev libsodium-dev
+
+OpenBSD
+~~~~~~~
+
+.. code-block:: bash
+
+    $ pkg_add sqlite3 geolite2-city geolite2-asn libsodium
+
+Gentoo
+~~~~~~
+
+.. code-block:: bash
+
+    emerge --ask sys-libs/libseccomp dev-db/sqlite dev-libs/libsodium
+
+Windows
+~~~~~~~
+
+You don't need to install any dependencies on windows, but you need to use a
+different build command in the next section.
+
+Building
+--------
+
+After all dependencies have been installed, simply build the binary:
+
+.. code-block:: bash
+
+    $ cargo build --release
+
+After the build finished the binary is located at ``target/release/sn0int``.

docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'sn0int'
-copyright = '2018, kpcyrd'
+copyright = '2018-2020, kpcyrd'
 author = 'kpcyrd'
 
 # The short X.Y version
@@ -138,7 +138,7 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('man', 'sn0int', 'OSINT framework and package manager',
+    ('man', 'sn0int', 'Semi-automatic OSINT framework and package manager',
      [author], 1)
 ]
 

docs/config.rst

@@ -0,0 +1,54 @@
+Configuration
+=============
+
+This section documents the config file. By default this file does not exist and
+a default configuration is used instead.
+
+Linux/BSD
+    ``~/.config/sn0int.toml``
+
+OSX
+    ``~/Library/Preferences/sn0int.toml``
+
+Windows
+    ``%APPDATA%/sn0int.toml``
+
+[core]
+------
+
+``registry``
+    Configure the registry you want to use. Defaults to ``https://sn0int.com``.
+``no-autoupdate``
+    sn0int is going to check if your modules are outdated during startout once
+    a week. Set this option to ``true`` to disable this.
+
+[namespaces]
+------------------
+
+By default sn0int modules are assumed to be installed from the registry. You
+may want to keep a local directory with private modules, especially during
+development. You can configure a folder that contains modules that aren't
+managed by sn0int by adding a namespace section to the config file::
+
+    [namespaces]
+    foo = "/opt/sn0int/foo"
+    bar = "~/repos/a/b/c/sn0int-modules"
+
+This is going to load modules from these two folders and register them in the
+``foo`` and ``bar`` namespace.
+
+Note that sn0int is also going to assume that symlinks in
+``~/.local/share/sn0int/modules`` and folders containing a ``.git`` folder are
+externally managed.
+
+[network]
+---------
+
+To enable a proxy, add the following to your config file::
+
+    [network]
+    proxy = "127.0.0.1:9050"
+
+This forces everything through tor (or any other socks5 proxy) and restricts
+all other functions that depend on the network. For example the ``dns``
+function is fully disabled if a proxy is configured.

docs/database.rst

@@ -46,6 +46,45 @@ triggered and an db_update is performed instead.
    removed from scope with ``noscope``. Everytime you use ``db_add`` you need
    to make sure that the ID that has been returned is not ``nil``.
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add_ttl('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_update
 ---------
 

docs/index.rst

@@ -1,20 +1,22 @@
 sn0int
 ======
 
-sn0int is an OSINT framework and package manager. It was built for IT security
-professionals and bug hunters to gather intelligence about a given target or
-about yourself. sn0int is enumerating attack surface by semi-automatically
-processing public information and mapping the results in a unified format for
-followup investigations.
+sn0int is a semi-automatic OSINT framework and package manager. It was built
+for IT security professionals and bug hunters to gather intelligence about a
+given target or about yourself. sn0int is enumerating attack surface by
+semi-automatically processing public information and mapping the results in a
+unified format for followup investigations.
 
 Among other things, sn0int is currently able to:
 
-- [X] Harvest subdomains from certificate transparency logs
-- [X] Harvest subdomains from various passive dns logs
-- [X] Sift through subdomain results for publicly accessible websites
-- [X] Harvest emails from pgp keyservers
-- [X] Enrich ip addresses with ASN and geoip info
-- [X] Harvest subdomains from the wayback machine
+- Harvest subdomains from certificate transparency logs
+- Harvest subdomains from various passive dns logs
+- Sift through subdomain results for publicly accessible websites
+- Harvest emails from pgp keyservers
+- Enrich ip addresses with ASN and geoip info
+- Harvest subdomains from the wayback machine
+- Gather information about phonenumbers
+- Bruteforce interesting urls
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource.  None of the investigations listed above are hardcoded
@@ -24,7 +26,7 @@ them with other users by publishing them to the sn0int registry. This allows
 you to ship updates for your modules on your own since you don't need to send a
 pull request.
 
-Join us on IRC: ircs://irc.hackint.org/#sn0int
+Join us on IRC: `irc.hackint.org:6697/#sn0int <https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int>`_
 
 Getting Started
 ---------------
@@ -34,7 +36,15 @@ Getting Started
    :glob:
 
    install
+   build
    usage
+   autonoscope
    scripting
    database
+   structs
+   activity
+   notifications
+   keyring
+   config
+   sandbox
    reference

docs/install.rst

@@ -1,7 +1,9 @@
 Installation
 ============
 
-If available, please prefer the package shipped by your linux distribution.
+If available, please prefer the package shipped by operating system. If your
+operating system has a package but you're running on older version, please use
+the `build from source <build.html>`_ instructions instead.
 
 Archlinux
 ---------
@@ -10,57 +12,102 @@ Archlinux
 
     $ pacman -S sn0int
 
-Debian/Ubuntu/Kali
-------------------
+Mac OSX
+-------
 
 .. code-block:: bash
 
-    $ apt install libsqlite3-dev libseccomp-dev
+    $ brew install sn0int
+
+Debian >= bullseye, Ubuntu >= 20.04, Kali
+-----------------------------------------
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+.. code-block:: bash
+
+    $ sudo apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg kpcyrd@archlinux.org | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ sudo apt update
+    $ sudo apt install sn0int
+
+Debian <= buster, Ubuntu <= 19.10
+---------------------------------
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+.. code-block:: bash
+
+    $ sudo apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | sudo apt-key add -
+    $ sudo apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ sudo apt update
+    $ sudo apt install sn0int
+
+Fedora/CentOS/Redhat
+--------------------
+
+Using rust+cargo from the repos might work for you, but we only officially
+support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
+at the docker image as an alternative.
+
+.. code-block:: bash
+
+    $ dnf install @development-tools libsq3-devel libseccomp-devel libsodium-devel publicsuffix-list
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ cargo install -f
+    $ cargo install -f --path .
+
+Docker
+------
+
+.. code-block:: bash
+
+    $ docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
 
 Alpine
 ------
 
 .. code-block:: bash
 
-    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f
+    $ apk add sn0int
 
 OpenBSD
 -------
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f
+    $ pkg_add sn0int
 
-Mac OSX
--------
+Gentoo
+------
 
 .. code-block:: bash
 
-    $ brew install sqlite3
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f
+    $ layman -a pentoo
+    $ emerge --ask net-analyzer/sn0int
+
+NixOS
+-----
+
+.. code-block:: bash
+
+    $ nix-env -i sn0int
 
 Windows
 -------
 
-This is not recommended and only passively maintained. Please prefer linux in a virtual machine if needed.
+This is not recommended and only passively maintained. Please prefer linux in a
+virtual machine if needed.
+
+Make sure rust is installed and setup.
 
 .. code-block:: bash
 
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-    $ 7z e sqlite3.zip -y
-    $ "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
-    $ export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
-    $ cargo install -f
+    $ cargo install -f --path .

docs/keyring.rst

@@ -0,0 +1,77 @@
+Keyring
+=======
+
+A common problem is that you need either an api key or a username/password
+combination. Instead of hardcoding it in the script you should request them
+from the keyring. In order to do this you need to request permissions to those
+credentials.
+
+Managing the keyring
+--------------------
+
+The keyring is a simple namespaced key-value store::
+
+    [sn0int][default] > keyring add aws:AKIAIOSFODNN7EXAMPLE
+    Secretkey: keep-this-secret
+    [sn0int][default] > keyring list
+    aws:AKIAIOSFODNN7EXAMPLE
+    [sn0int][default] >
+    [sn0int][default] > keyring list aws
+    aws:AKIAIOSFODNN7EXAMPLE
+    [sn0int][default] > keyring list instagram
+    [sn0int][default] >
+    [sn0int][default] > keyring get aws:AKIAIOSFODNN7EXAMPLE
+    Namespace:    "aws"
+    Access Key:   "AKIAIOSFODNN7EXAMPLE"
+    Secret:       "keep-this-secret"
+    [sn0int][default] >
+
+If the service uses a username-password combination, set the username as the
+access key and the password as the secret.
+
+If the service uses only a secret key for the api, set the secret key as the
+access key and leave the secret blank.
+
+A script doesn't automatically get access to requested keyring namespaces.
+Instead the user is asked to confirm those requests to limit abusive scripts.
+
+Using access keys in scripts
+----------------------------
+
+We can request all keys of a certain namespace in our script metadata. This is
+going to prompt the user to grant the script access. This can be done for
+multiple namespaces in the same script:
+
+.. code-block:: lua
+
+    -- Keyring-Access: aws
+    -- Keyring-Access: asdf
+
+If the user granted us access to those keys we can read them with ``keyring``:
+
+.. code-block:: lua
+
+    creds = keyring('aws')
+    debug(creds[1]['access_key'])
+    debug(creds[1]['secret_key'])
+
+This returns a list of all keys in that namespace. Any empty list is returned
+if the user doesn't have any keys in that namespace.
+
+If you want to allow the user to select a specific script you can introduce an
+option that is set by the user and then filter ``creds`` until the
+``access_key`` matches.
+
+Using access keys as source argument
+------------------------------------
+
+We can also use the access keys as source argument. This is useful if each
+account has access to different things and we want to read through all of them.
+
+Since access key permissions are granted per namespace we need to specify which
+credentials we want to use.
+
+.. code-block:: lua
+
+    -- Keyring-Access: aws
+    -- Source: keyring:aws

docs/man.rst

@@ -10,4 +10,5 @@ todo
     :glob:
 
     usage
+    config
     reference

docs/notifications.rst

@@ -0,0 +1,311 @@
+Notifications
+=============
+
+If you run sn0int unattended nobody might see the sn0int output. For cases like
+this you can configure notifications to send you a push notification in case
+something interesting happens. This is also especially useful if you have
+sn0int setup to run automatically.
+
+Receiving notifications
+-----------------------
+
+Notifications are just regular sn0int modules. You can install them just like
+any other module or write your own. This section contains walkthroughs on how
+to setup common integrations.
+
+Telegram
+~~~~~~~~
+
+Install the telegram notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-telegram
+
+Open your telegram app and open a chat with ``@botfather``. Send ``/newbot``
+and answer the questions. Copy ``bot_token`` and open this url in your browser:
+
+.. code-block::
+
+    https://api.telegram.org/bot**your_bot_token**/getUpdates
+
+Back on your app, open the t.me link to start a new chat with your bot, then
+send ``/start``. Reload the page in your browser, you should see the new
+message you sent. Copy the ``chat_id``.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-telegram -o bot_token=1337:foobar -o chat_id=1337 'hello world'
+
+You should receive ``hello world`` from your bot on Telegram.
+
+Pushover
+~~~~~~~~
+
+Install the pushover notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-pushover
+
+Signup for pushover and configure the app on your device. Copy th user key
+visible on the pushover dashboard. Click "Create an Application/API Token". Set
+"sn0int" as name and set an icon if you want to. Copy the api token.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-pushover -o user_key=asdf1337 -o api_token=asdf1337 'hello world'
+
+You should receive ``hello world`` as a push notification.
+
+Discord
+~~~~~~~
+
+Install the discord notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-discord
+
+Decide which channel should receive notifications (or create a new one). Open
+the "Server Settings" of your discord server. Click on "Webhooks". Click
+"Create Webhook". Configure the Name and Channel. Copy the Webhook URL.
+
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-discord -o url=https://discord.com/api/webhooks/1337/asdf 'hello world'
+
+You should receive ``hello world`` in your discord channel.
+
+Signal
+~~~~~~
+
+Install the sn0int notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-signal
+
+This module allows end-to-end encrypted notifications, but it's also difficult
+to setup. You need a second phone number and install both `signal-cli
+<https://github.com/AsamK/signal-cli>`_ and `sn0int-signal
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+After you've registered your second phone number with signal-cli, you can use
+sn0int-signal to expose a minimal api for notify-signal. For more detailed
+instructions and how to start the api at boot, see the `sn0int-signal README
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+Read the secret key generated at ``/etc/sn0int-signal.key`` and send a
+notification to the signal phone number:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-signal -o to=+31337 -o secret=asdf 'hello world'
+
+You should receive ``hello world`` from the number signed up with signal-cli.
+
+Writing your own module
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Make sure you've read the detailed instructions on how to get setup with
+`module development <scripting.html>`_.
+
+Create a new sn0int module like this:
+
+.. code-block:: bash
+
+    sn0int new ~/repos/sn0int-modules/notify-custom.lua
+
+Edit the ``-- Source:`` so it takes notifications as input:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        debug(arg)
+        info(arg['subject'])
+        info(arg['body'])
+    end
+
+Execute your script:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom 'hello world'
+
+You most likely need to pass options to avoid hard-coding keys into your
+script. Options can be fetched like this:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        local foo = getopt('foo')
+        if not foo then return 'Missing -o foo= option' end
+
+        info('foo: ' .. foo)
+        info('subject: ' .. arg['subject'])
+    end
+
+And passed like this:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom -o "foo=hello world" 'ohai'
+
+Setting up notification rules
+-----------------------------
+
+We now know how to trigger notifications manually, but we would rather trigger
+notifications if a module runs into something interesting.
+
+You can setup subscriptions on specific topics and then have a notification
+script execute automatically.
+
+Lookup the location of your sn0int config file:
+
+.. code-block:: bash
+
+    sn0int paths
+
+And open it in an editor of your choice:
+
+.. code-block:: bash
+
+    vim /home/user/.config/sn0int.toml
+
+A basic configuration could look like this:
+
+.. code-block:: toml
+
+    # You can have multiple notification sections, this one is named
+    # `demo-telegram-integration`
+    # The label can be set to whatever you want, but you may need to add
+    # double-quotes to use some characters.
+    [notifications.demo-telegram-integration]
+    # If this option is present, the notification must originate from one of
+    # the following workspaces.
+    workspaces = ["default", "some-workspace"]
+    # If this option is present, the notification must match one of the
+    # filters. You can use `*` as a wildcard to match everything except `:`.
+    topics = ["activity:harness/activity-ping:*"]
+    # Mandatory: the module to execute.
+    script = "kpcyrd/notify-telegram"
+    # The options to pass to the module, if any.
+    # Can be accessed with `getopt`
+    options = [
+        "bot_token=1337:foobar",
+        "chat_id=1337",
+    ]
+
+All options except ``script`` are optional, but setting filters is highly
+recommended.
+
+Testing notifications
+---------------------
+
+To test if your configuration works correctly you can create an event manually:
+
+.. code-block:: bash
+
+    sn0int -w some-workspace notify send activity:harness/activity-ping:dummy "hello world"
+
+If it matches any of your rules you should receive a push notifications.
+
+.. note::
+    If you want to test just the routing without actually sending something, add ``--dry-run``.
+
+Running sn0int automatically
+----------------------------
+
+Support for this is going to improve in the future, but you can already set
+this up if you're ok with a slightly buggy experience.
+
+Monitors
+~~~~~~~~
+
+Some modules are long-running and either wait for an event from a server or
+have custom polling built in that's usually configurable with an ``-o
+interval=`` option. If your module has a non-trivial setup phase, an author may
+take this approach.
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-new-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+    Restart=always
+    RestartSec=0
+
+    [Install]
+    WantedBy=multi-user.target
+
+Enable the service to run on boot:
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-new-service.service
+
+Timers
+~~~~~~
+
+If the module is only one-shot you can set it up to run with a timer:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+Setup the timer like this:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.timer
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Timer]
+    OnBootSec=1min
+    OnUnitActiveSec=1h
+
+    [Install]
+    WantedBy=timers.target
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-other-service.timer

docs/sandbox.rst

@@ -0,0 +1,147 @@
+Sandbox
+=======
+
+Scripts are generally considered to be untrusted and executed exclusively in a
+child process. It's important to note that there's a basic sandbox that's
+active on every operating system, and there's a second line of defense on
+supported operating systems.
+
+The first line of defense is the restrictive stdlib. It's assumed that an
+attacker gains full control over the lua code and is able to call any function
+with arbitrary arguments. The stdlib only provides functions that are
+considered safe, so for example it's not possible to start a process or open a
+file.
+
+The second line of defense is supposed to make sure the system isn't
+compromised even if the first layer is fully broken and an attacker gains full
+control over the child process.
+
+Right now this is only supported on linux and openbsd.
+
+Linux
+-----
+
+On linux we use seccomp to filter all syscalls that we don't need. We also use
+chroot to disable filesystem access. It's recommended to install the sn0int
+binary with ``cap_sys_chroot`` to make sure unprivileged users can use chroot.
+The chroot location is hard coded and all capabilities are removed after the
+chroot is done or if no chroot is going to happen.
+
+OpenBSD
+-------
+
+On openbsd we're using ``pledge`` to restrict syscalls and ``unveil`` to
+restrict filesystem access.
+
+IPC Protocol
+------------
+
+The parent process and the child process communicate using an IPC protocol that
+is line-based json.
+
+For a simple hello world the parent process is only going to send a single line
+to the child process. This line contains:
+
+- The function argument
+- The dns config
+- Keys that the module has been given access to
+- The module metadata and code
+- Options, if any
+- A socks5 proxy, if any
+- The log level
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    -- nothing to do here\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+
+Saving this line in a file called ``start.json`` and sending it to a sandbox
+process should result in the following output::
+
+    $ sn0int sandbox foobar < start.json
+    {"Exit":"Ok"}
+    $
+
+This line tells us that the script terminated successfully.
+
+There are some functions that cause a notification to the parent process. We
+are going to add a call to the ``info()`` function to our module:
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    info('ohai')\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+
+This is going to print an additional event::
+
+    $ sn0int sandbox foobar < start2.json
+    {"Log":{"Info":"\"ohai\""}}
+    {"Exit":"Ok"}
+    $
+
+There are some functions that block the child process until the parent process
+sent a reply. These functions are mostly database related functions, since the
+child doesn't have direct database access. To demonstrate this, we're going to
+write two lines to our file this time, one is the init line and the second one
+is the reply for the database event:
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    x = db_add('domain', {value=\"example.com\"})\n    info(x)\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+    {"Ok":1337}
+
+Results in the following output::
+
+    $ target/release/sn0int sandbox foobar < start3.json
+    {"Database":{"Insert":{"Domain":{"value":"example.com"}}}}
+    {"Log":{"Info":"1337.0"}}
+    {"Exit":"Ok"}
+    $
+
+The first line is a database event and indicates that the child wants to insert
+data. After printing this line the child tries to read a line from stdin, this
+is why we needed to write two lines to our json file this time. In the second
+line the child learns if the insert was successful and which id was assigned to
+that entity.
+
+Limitations
+-----------
+
+There are some limitations that you should be aware:
+
+- Network access is available and network namespaces aren't isolated. This
+  means scripts have access to your local network, the internet and also your
+  localhost loopback interface.
+- If chroot is unavailable an attacker could connect to unix domain sockets.
+
+Diagnosing a sandbox failure
+----------------------------
+
+You might experience a sandbox failure, especially on architectures that are
+less popular. This usually looks like this::
+
+    [sn0int][example][kpcyrd/ctlogs] > run
+    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [+] Finished kpcyrd/ctlogs (1 errors)
+
+A module that never finishes could also mean an IO thread inside the worker got
+killed by the sandbox.
+
+You can try to diagnose this yourself with strace::
+
+    strace -f sn0int run -vv ctlogs 2>&1 | tee strace.log
+
+Open ``strace.log``, look out for syscalls that didn't return by searching for
+``= ?`` and ignore calls to exit and similar. You are looking for something
+like this::
+
+    seccomp(SECCOMP_SET_MODE_FILTER, 0, {len=48, filter=0xdd59094e490}) = 0
+    write(1, "[+] activated!\n", 15[+] activated!
+    )        = 15
+    getresuid( <unfinished ...>)            = ?
+    +++ killed by SIGSYS (core dumped) +++
+
+This would indicate a call to ``getresuid`` which was not allowed by the
+seccomp filter.
+
+If you don't want to diagnose this yourself open a new bug report with as much
+information as possible, specifically which distro, which release and which
+architecture you're using.

docs/scripting.rst

@@ -1,11 +1,11 @@
-Scripting
-=========
+Writing your first module
+=========================
 
 Scripting is the core feature in sn0int. It's not strictly required, but if you
 want to write your own modules, this section is for you.
 
-Write your first module
------------------------
+Creating a repository
+---------------------
 
 It's highly recommended to use a VCS for development, so let's start by setting
 that up. We're going to assume you store your repos in ``~/repos`` but you're
@@ -13,13 +13,35 @@ free to change that to something else::
 
     $ git init ~/repos/sn0int-modules
     $ cd ~/repos/sn0int-modules
-    $ ln -s "$PWD" ~/.local/share/sn0int/modules/$YOUR_GITHUB_NAME
+
+.. note::
+   If you're using github you can also create a repo from the `module repo
+   template`_.
+
+.. _module repo template: https://github.com/sn0int/sn0int-modules
+
+We need to add this folder to the sn0int config file so it's correctly detected
+when starting sn0int. Open the `config file <config.html>`_ in your prefered
+editor. Note that the file does not exist by default and the path is different
+depending on your operating system. On linux you would open the config file
+with::
+
+    $ vim ~/.config/sn0int.toml
+
+Add the following::
+
+    [namespaces]
+    your_github_name = "~/repos/sn0int-modules"
 
 Every module we're adding to ``~/repos/sn0int-modules`` is now going to be
 picked up by sn0int.
 
-Let's add our first module by opening ``~/repos/sn0int-modules/first.lua``.
-There's a bit of boilerplate that every module needs to load successfully:
+Make sure you're still in the right folder and add your first module::
+
+    sn0int new first.lua
+
+This is going to generate some boilerplate for you that every module needs to
+load successfully. Afterwards we can edit it like this:
 
 .. code-block:: lua
 
@@ -74,21 +96,26 @@ database.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
-        print(subdomain)
+        info(subdomain)
     end
 
-Combined with the header we wrote previously we can already execute this
-module. Make sure you've added a domain to scope with ``add domain
-example.com``, save your file and run it like this::
+This is already enough to execute it. Make sure you've added a domain to scope
+with ``add domain example.com``, save your file and run it like this::
 
     sn0int run -f ./first.lua
 
-We should see some output by our print function.
+We should see some output by our info function.
 
 .. note::
-   ``print`` is useful for development but must be removed before publishing.
+   ``info`` is useful for development but you usually want your module to run
+   quietly, so before publishing either remove it or replace it with ``debug``.
 
 Next, we want to actually resolve that name, we're going to use the ``dns``
 function for that. This function takes a name and a query type and returns a
@@ -98,29 +125,45 @@ truth-y.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
-        records = dns(subdomain, 'A')
+        records = dns(subdomain, {
+            record='A'
+        })
         if last_err() then return end
 
-        print(records)
+        info(records)
     end
 
 If you run your module again you're going to see some output, either
-``{"success": somedata}`` or ``{"error": "NX"}``. We decide if ``success`` is
-not ``nil``, we add the subdomain to our scope and set it to resolvable:
+``{"answers":[somedata],"error":null}`` or
+``{"answers":[],"error":"NXDomain"}``. If the dns reply doesn't indicate an
+error this means the subdomain exists and we can add it to our database with
+``resolvable`` being set to ``true``.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
-        records = dns(subdomain, 'A')
+        records = dns(subdomain, {
+            record='A'
+        })
         if last_err() then return end
 
-        if records['success'] ~= nil then
-            db_add('subdomain', arg, {
+        if records['error'] == nil then
+            db_add('subdomain', {
                 domain_id=arg['id'],
                 value=subdomain,
                 resolvable=true,
@@ -131,39 +174,20 @@ not ``nil``, we add the subdomain to our scope and set it to resolvable:
 .. hint::
    See the database section to understand how the database works in detail.
 
-If we execute our module one more time it's going to log that it discovered a
-subdomain, if it doesn't, try adding more domains to scope. Note that this only
-happens the first time. Modules that don't discover anything or don't discover
-anything new exit silently.
-
-After putting everything together, our final module looks like this:
-
-.. code-block:: lua
-
-    -- Description: ohai wurld
-    -- Version: 0.1.0
-    -- Source: domains
-    -- License: GPL-3.0
-
-    function run(arg)
-        subdomain = 'www.' .. arg['value']
-
-        records = dns(subdomain, 'A')
-        if last_err() then return end
-
-        if records['success'] ~= nil then
-            db_add('subdomain', arg, {
-                domain_id=arg['id'],
-                value=subdomain,
-                resolvable=true,
-            })
-        end
-    end
+If we execute our finished module one more time it's going to log that it
+discovered a subdomain, if it doesn't, try adding more domains to scope. Note
+that this only happens the first time. Modules that don't discover anything or
+don't discover anything new exit silently.
 
 There's still some room for improvement, for example, since we already resolved
 that record, we could also add the ip address to the scope and link it to the
 subdomain we added.
 
+.. hint::
+   For debugging purposes you can increase the verbosity with ``sn0int run -v``
+   so database operations are logged even if nothing was changed, or with
+   ``sn0int run -vv`` to enable ``debug()`` output.
+
 Publish your module
 -------------------
 
@@ -181,3 +205,58 @@ your identity.
 Afterwards publish your module with::
 
     sn0int publish ./first.lua
+
+Please also make sure you publish your repository to github so other people can
+submit pull requests. The recommended repository location is::
+
+    https://github.com/<your-username>/sn0int-modules
+
+Publish your repo
+-----------------
+
+It is highly recommended to publish your repository on github so people can
+file issues and pull requests for your module. If you've been following along
+with the github template you can simply commit your changes and push them.
+
+Your repository would look like one of these:
+
+- https://github.com/kpcyrd/sn0int-modules
+- https://github.com/ysf/sn0int-modules
+- https://github.com/cybiere/sn0int-modules
+
+Reading data from stdin
+-----------------------
+
+Sometimes you need to read data that can't be easily accessed from within the
+sandbox, like output of other programms or file content. In that case you can
+write a module that reads from stdin:
+
+.. code-block:: lua
+
+    -- Description: Read from stdin
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+
+    function run()
+        while true do
+            x = stdin_readline()
+            if x == nil then
+                break
+            end
+            info(x)
+        end
+    end
+
+Write it to a file and run it like this::
+
+    % echo hello | sn0int run --stdin -vvf stdin.lua
+    [*] anonymous/stdin                                   : "hello\n"
+    [+] Finished anonymous/stdin
+    %
+
+This is going to read one line at a time and allows you to process it with
+regular expressions and add data to the database.
+
+.. note::
+   If you get an error like ``Failed to read stdin: "stdin is unavailable"``
+   make sure the ``--stdin`` flag is set.

docs/structs.rst

@@ -0,0 +1,335 @@
+Structs
+=======
+
+This section describes all supported structs in depth. Please refer to this
+section if in doubt about the correct usage of fields to ensure
+interoperability between modules.
+
+Domains
+-------
+
+Represents a registerable domain as defined by the `public suffix list
+<https://publicsuffix.org/>`_. If in doubt check `psl_domain_from_dns_name
+<reference.html#psl-domain-from-dns-name>`_.
+
+``value``
+  The domain name, like ``example.co.uk``.
+
+Subdomains
+----------
+
+A subdomain of a `domain <#domains>`_. The depth is arbitrary, so
+``foo.example.co.uk`` and ``foo.bar.example.co.uk`` are both valid subdomains
+of ``example.co.uk``.
+
+``value``
+  The subdomain, like ``foo.bar.example.co.uk``.
+``domain_id``
+  The numeric id of a domain struct.
+``resolvable``
+  Whether the subdomain can be resolved to a A/AAAA record. nil if unknown.
+
+IpAddrs
+-------
+
+An ip address. Note that most of these fields are geoip related and an
+approximation instead of an actual location.
+
+``value``
+    The ip address.
+``family``
+    The address family of the ip address, either ``4`` or ``6``.
+``continent``
+    The continent associated with this ip address.
+``continent_code``
+    The continent code of the ``continent`` field, eg ``NA``.
+``country``
+    The country associated with this ip address.
+``country_code``
+    The country code of the ``country`` field, eg ``US``.
+``city``
+    The city associated with this ip address.
+``latitude``
+    Latitude associated with this ip address.
+``longitude``
+    Longitude associated with this ip address.
+``asn``
+    The number of the autonomous system this ip belongs to.
+``as_org``
+    The organization of the autonomous system this ip belongs to.
+``description``
+    This field is sn0int internal if we have additional information about this
+    ip address, for example technical identifiers from aws.
+``reverse_dns``
+    The reverse dns name setup for this ip address.
+
+URLs
+----
+
+``subdomain_id``
+    The numeric id of a subdomain struct.
+``value``
+    The url, including a schema, hostname and path.
+``status``
+    The http status code, like ``200``.
+``body``
+    The raw response body. This can be any mime type.
+``online``
+    Whether or not the url gives a http response (even if it's an error).
+``title``
+    The parsed ``<title>`` of the page, if available.
+``redirect``
+    If the server replied with a redirect, this is the url it redirected to.
+
+Emails
+------
+
+``value``
+    The email address.
+``displayname``
+    The display name of a given email address: ``this is the name <foo@example.com>``.
+``valid``
+    Whether that email address is valid or has been disabled.
+
+Phonenumbers
+------------
+
+``value``
+    The phone number in E.164 format (+491234567)
+``name``
+    An alias we can assign to this phone number. This alias is sn0int internal.
+``valid``
+    Whether the number is assigned to a customer.
+``last_online``
+    The last time this number has been online.
+``country``
+    The country this number is associated with.
+``carrier``
+    The name of the carrier this numer is registered with.
+``line``
+    The type of the phone number, can be ``landline``, ``mobile`` or ``voip``.
+``is_ported``
+    Whether this number has been ported to a different carrier.
+``last_ported``
+    The last time this number has been ported.
+``caller_name``
+    The name of the owner of the phone number.
+``caller_type``
+    The type of caller, eg ``business`` or ``consumer``.
+
+Devices
+-------
+
+``value``
+    The devices mac address or another identifier if needed.
+``name``
+    An alias we can assign to this device. This alias is sn0int internal.
+``hostname``
+    The hostname configured on the device.
+``vendor``
+    The hardware vendor of the device. This is usually derived from the mac
+    address.
+``last_seen``
+    The last time we've observed the device somewhere.
+
+Networks
+--------
+
+A wired or wireless network at a specific location that a device could be
+connected to.
+
+``value``
+    The network name. This can be an ssid or any other identifier but should be
+    unique.
+``latitude``
+    Latitude of the networks location.
+``longitude``
+    Longitude of the networks location.
+``description``
+    A human readable description in case the value is a technical identifier.
+
+Accounts
+--------
+
+A users account or profile on a webservice, like github or instagram.
+
+``service``
+    The identifier of the service/website. It's recommended to use the websites
+    domain for this as defined in `Domains`_.
+``username``
+    The users unique identifier, like the login name. If the login name is not
+    known or the system doesn't use login names, use the email address instead.
+``displayname``
+    The users display name. This name is often not unique and may contain the
+    users real name.
+``email``
+    The email address associated with the account.
+``url``
+    The url of the public profile if available.
+``last_seen``
+    The last time this account has been active/online.
+``birthday``
+    The users birthday set on the account.
+``phonenumber``
+    The phonenumber associated with the account.
+``profile_pic``
+    The blob identifier of the users current profile picture.
+
+Breaches
+--------
+
+Either a breach of a specific website, a breach compilation or a breach
+notification service.
+
+``value``
+    The name of the breach, breach compilation or notification service.
+
+Images
+------
+
+``value``
+    The id that identifies the blob. This id is deterministic based on file
+    content.
+``filename``
+    This field is used if we have a well known filename for the content.
+``mime``
+    The image mimetype, like ``image/png`` or ``image/jpeg``.
+``width``
+    The width of the image.
+``height``
+    The height of the image.
+``created``
+    The date and time this image has been taken.
+``latitude``
+    Latitude this picture has been taken.
+``longitude``
+    Longitude this picture has been taken.
+``nudity``
+    A score that classifies nudity in this picture. The score goes from 0 to 2
+    and is commonly calculated with ``img_nudity``. A score above 1 means
+    nudity has been detected.
+``ahash``
+    The Mean (aHash) perceptual hash.
+``dhash``
+    The Gradient (dHash) perceptual hash.
+``phash``
+    The DCT (pHash) perceptual hash.
+
+Ports
+-----
+
+The status of a port on an ip address.
+
+``ip_addr_id``
+    The numeric id of an ipaddr struct.
+``ip_addr``
+    The actual ipaddr.
+``port``
+    The port number.
+``status``
+    The status of the port, either ``open`` or ``closed``.
+``banner``
+    The service banner we discovered on this port.
+``service``
+    The service that is running on this port.
+``version``
+    The version of the service running on this port.
+
+Netblocks
+---------
+
+A netblock is a network address range that has been allocated to an individual,
+organization or company. Those are commonly found when running whois lookups on
+an ip address.
+
+Consider the following example: Running a whois lookup on ``140.82.118.4`` (one
+of the addresses currently in use by github) returns that this address belongs
+to the netrange ``140.82.112.0 - 140.82.127.255``, so the netblock in this case
+is ``140.82.112.0/20``.
+
+``family``
+    This is either ``4`` or ``6`` and populated automatically.
+``value``
+    This is the network range in CIDR notation.
+``asn``
+    The number of the autonomous system this network belongs to.
+``as_org``
+    The organization of the autonomous system this network belongs to.
+``description``
+    This field isn't strictly defined and meant to be used as a human
+    meaningful name if available.
+
+CryptoAddrs
+-----------
+
+A cryptoaddr is any cryptocurrency address and not tied to a specific currency.
+
+``value``
+    The address string. This looks like ``1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2``.
+``currency``
+    The identifier for a specific currency. This is usually the ticker symbols,
+    like ``xbt``, ``zec`` or ``xmr``.
+``denominator``
+    Balance is tracked internally using 64 bit integers (signed, for technical reasons). Balance is supposed to be the lowest unit, so in case of bitcoin you'd write ``100,000,000`` satoshi instead of ``1`` bitcoin. Since this value is inconvinient to work with we're using the denominator to display values. In case of bitcoin you'd set it to ``8``.
+``balance``
+    The current balance of the address, in the lowest possible unit. In case of bitcoin this would be satoshis.
+``received``
+    The total amount of currency received by this address.
+``first_seen``
+    The first time currency was sent to this address.
+``last_withdrawal``
+    The last time a transaction signed by this address was observed.
+``description``
+    A human readable note for this address.
+
+Activity
+--------
+
+Activity is different from all other structs, have a look at the `Activity
+Section <activity.html>`_.
+
+Relations
+---------
+
+Relations are linking two structs together. The link may contain additional information.
+
+subdomain_ipaddr
+~~~~~~~~~~~~~~~~
+
+Links an ip address to a subdomain.
+
+``subdomain_id``
+    The numeric id of a subdomain struct.
+``ip_addr_id``
+    The numeric id of an ip addr struct.
+
+network_device
+~~~~~~~~~~~~~~
+
+Links a device to a network. This is commonly used with ``db_add_ttl`` so the
+link automatically expires. This is frequently used to monitor networks for
+known and unknown devices.
+
+``network_id``
+    The numeric id of a network struct.
+``device_id``
+    The numeric id of a device struct.
+``ipaddr``
+    The ip address assigned to the device.
+``last_seen``
+    The last time we've seen the device on that network.
+
+breach_email
+~~~~~~~~~~~~
+
+Links an email to a breach. If we know the password as well we can add it to
+the link. If we don't know the password we can leave it blank and fill it
+later. An email can be linked to a breach multiple times with different
+passwords. There is a special upserting logic in place to support this.
+
+``breach_id``
+    The numeric id of a breach struct.
+``email_id``
+    The numeric id of an email struct.
+``password``
+    The password for that email in the breach.

docs/usage.rst

@@ -27,12 +27,12 @@ number of recommended modules::
     [+] Downloading "GeoLite2-City.mmdb"
     [+] Downloading "GeoLite2-ASN.mmdb"
     [+] Loaded 0 modules
-    [*] No modules found, run quickstart to install default modules
+    [*] No modules found, run pkg quickstart to install default modules
     [sn0int][default] >
 
-Typing ``quickstart`` is going to get you a fair number of featured modules::
+Typing ``pkg quickstart`` is going to get you a fair number of featured modules::
 
-    [sn0int][default] > quickstart
+    [sn0int][default] > pkg quickstart
     [+] Installing kpcyrd/asn
     [+] Installing kpcyrd/ctlogs
     [+] Installing kpcyrd/dns-resolve
@@ -105,7 +105,7 @@ Running a module
 Now that we have something to get started with, we can run our first module.
 First lets list all modules we have::
 
-    [sn0int][demo] > mod list
+    [sn0int][demo] > pkg list
     kpcyrd/asn (0.1.0)
         Run a asn lookup for an ip address
     kpcyrd/ctlogs (0.1.0)
@@ -149,7 +149,7 @@ some of them in a browser but hold on, there's a more efficient way to approach
 this.
 
 .. hint::
-   You can run the modules concurrently with ``run -j 8``.
+   You can run the modules concurrently with ``run -j3``.
 
 Running followup modules on the results
 ---------------------------------------

examples/asn.rs

@@ -1,32 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate maxminddb;
-
-use std::env;
-use sn0int::errors::*;
-use sn0int::geoip::{AsnDB, Maxmind};
-
-
-fn run() -> Result<()> {
-    let asndb = AsnDB::open_or_download()?;
-
-    for arg in env::args().skip(1) {
-        let ip = arg.parse()?;
-        let asn = asndb.lookup(ip)?;
-        println!("{:#?}", asn);
-    }
-
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}

examples/geoip.rs

@@ -1,31 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-
-use std::env;
-use sn0int::errors::*;
-use sn0int::geoip::{GeoIP, Maxmind};
-
-
-fn run() -> Result<()> {
-    let geoip = GeoIP::open_or_download()?;
-
-    for arg in env::args().skip(1) {
-        let ip = arg.parse()?;
-        let lookup = geoip.lookup(ip)?;
-        println!("{:#?}", lookup);
-    }
-
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}

examples/maxmind-dl.rs

@@ -1,50 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
-use sn0int::errors::*;
-use sn0int::geoip::{GeoIP, Maxmind};
-use sn0int::paths;
-use std::fs;
-use structopt::StructOpt;
-
-#[derive(Debug, StructOpt)]
-pub struct Args {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-fn run() -> Result<()> {
-    let args = Args::from_args();
-    debug!("{:?}", args);
-    let path = paths::cache_dir()?.join(&args.target);
-    if args.extract_only {
-        let body = fs::read(&args.url)?;
-        sn0int::archive::extract(&mut &body[..], &args.filter, path)?;
-    } else {
-        GeoIP::download(path, &args.filter, &args.url)?;
-    }
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}

examples/maxmind.rs

@@ -0,0 +1,71 @@
+use sn0int::errors::*;
+use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
+use sn0int::paths;
+use std::net::IpAddr;
+use std::path::Path;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="asn")]
+    Asn(AsnArgs),
+    #[structopt(name="geoip")]
+    GeoIP(GeoIPArgs),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AsnArgs {
+    ip: IpAddr,
+}
+
+impl AsnArgs {
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = AsnDB::cache_path(cache_dir)?;
+        let asndb = AsnDB::open(&path)?;
+
+        let asn = asndb.lookup(self.ip)?;
+        println!("{:#?}", asn);
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct GeoIPArgs {
+    ip: IpAddr,
+}
+
+impl GeoIPArgs {
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = GeoIP::cache_path(cache_dir)?;
+        let geoip = GeoIP::open(&path)?;
+
+        let lookup = geoip.lookup(self.ip)?;
+        println!("{:#?}", lookup);
+
+        Ok(())
+    }
+}
+
+
+fn run() -> Result<()> {
+    let args = Args::from_args();
+    debug!("{:?}", args);
+    let cache_dir = paths::cache_dir()?;
+    match args {
+        Args::Asn(args) => args.run(&cache_dir),
+        Args::GeoIP(args) => args.run(&cache_dir),
+    }
+}
+
+fn main() {
+    env_logger::init();
+
+    if let Err(err) = run() {
+        eprintln!("Error: {}", err);
+        for cause in err.iter_chain().skip(1) {
+            eprintln!("Because: {}", cause);
+        }
+        std::process::exit(1);
+    }
+}

examples/spinners.rs

@@ -1,20 +1,67 @@
-extern crate sn0int;
-
-use std::env;
 use std::thread;
 use std::time::Duration;
-use sn0int::term::{SPINNERS, Spinner};
+use sn0int::term::{SPINNERS, Spinner, StackedSpinners};
+use structopt::StructOpt;
+
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="single")]
+    Single(Single),
+    #[structopt(name="stacked")]
+    Stacked(Stacked),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Single {
+    idx: usize,
+    #[structopt(long="ticks", default_value="100")]
+    ticks: usize,
+}
+
+impl Single {
+    fn run(&self) {
+        let mut s = Spinner::new(SPINNERS[self.idx], "Demo".to_string());
+
+        for _ in 0..self.ticks {
+            thread::sleep(Duration::from_millis(100));
+            s.tick();
+        }
+
+        s.finish("Done".to_string());
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Stacked {
+}
+
+impl Stacked {
+    fn run(&self) {
+        let mut stack = StackedSpinners::new();
+        stack.add("1".into(), String::from("spinner1"));
+        stack.add("2".into(), String::from("spinner2"));
+        stack.add("3".into(), String::from("spinner3"));
+
+        for x in 1..=3 {
+            for _ in 0..50 {
+                thread::sleep(Duration::from_millis(100));
+                stack.tick();
+            }
+            // stack.log("ohai");
+            stack.remove(&x.to_string());
+        }
+
+        stack.clear();
+
+        // stack.finish("Done".to_string());
+    }
+}
 
 fn main() {
-    let idx = env::args().skip(1).next().expect("Expected argv[1]");
-    let idx = idx.parse::<usize>().expect("argv[1] is not a number");
-
-    let mut s = Spinner::new(SPINNERS[idx], "Demo".to_string());
-
-    for _ in 0..100 {
-        thread::sleep(Duration::from_millis(100));
-        s.tick();
+    let args = Args::from_args();
+    match args {
+        Args::Single(args) => args.run(),
+        Args::Stacked(args) => args.run(),
     }
-
-    s.finish("Done".to_string());
 }

examples/stacked-spinners.rs

@@ -1,25 +0,0 @@
-extern crate sn0int;
-
-use sn0int::term::StackedSpinners;
-use std::thread;
-use std::time::Duration;
-
-fn main() {
-    let mut stack = StackedSpinners::new();
-    stack.add("1".into(), String::from("spinner1"));
-    stack.add("2".into(), String::from("spinner2"));
-    stack.add("3".into(), String::from("spinner3"));
-
-    for x in 1..=3 {
-        for _ in 0..50 {
-            thread::sleep(Duration::from_millis(100));
-            stack.tick();
-        }
-        // stack.log("ohai");
-        stack.remove(&x.to_string());
-    }
-
-    stack.clear();
-
-    // stack.finish("Done".to_string());
-}

migrations/2018-10-26-001932_cascade-delete/up.sql

@@ -37,25 +37,6 @@ INSERT INTO subdomains (id, domain_id, value, unscoped, resolvable)
 
 DROP TABLE _subdomains_old;
 
--- subdomain_ipaddrs
-
-ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
-
-CREATE TABLE subdomain_ipaddrs (
-    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-    subdomain_id INTEGER NOT NULL,
-    ip_addr_id INTEGER NOT NULL,
-    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
-    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
-    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
-);
-
-INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
-    SELECT id, subdomain_id, ip_addr_id
-    FROM _subdomain_ipaddrs_old;
-
-DROP TABLE _subdomain_ipaddrs_old;
-
 -- urls
 
 ALTER TABLE urls RENAME TO _urls_old;
@@ -123,4 +104,23 @@ INSERT INTO ipaddrs (id, family, value, unscoped, continent, continent_code, cou
 
 DROP TABLE _ipaddrs_old;
 
+-- subdomain_ipaddrs
+
+ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
+
+CREATE TABLE subdomain_ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
+);
+
+INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
+    SELECT id, subdomain_id, ip_addr_id
+    FROM _subdomain_ipaddrs_old;
+
+DROP TABLE _subdomain_ipaddrs_old;
+
 PRAGMA foreign_keys=on;

migrations/2018-12-03-143558_url_path/down.sql

@@ -0,0 +1,25 @@
+PRAGMA foreign_keys=off;
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    title VARCHAR,
+    redirect VARCHAR,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online, title, redirect)
+  SELECT id, subdomain_id, value, status, body, unscoped, online, title, redirect
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-03-143558_url_path/up.sql

@@ -0,0 +1,26 @@
+PRAGMA foreign_keys=off;
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    path VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    title VARCHAR,
+    redirect VARCHAR,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, path, status, body, unscoped, online, title, redirect)
+  SELECT id, subdomain_id, value, '/', status, body, unscoped, online, title, redirect
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-14-094011_phonenumbers/down.sql

@@ -0,0 +1 @@
+DROP TABLE phonenumbers;

migrations/2018-12-14-094011_phonenumbers/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE phonenumbers (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    name VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    last_online DATETIME,
+    country VARCHAR,
+    carrier VARCHAR,
+    line VARCHAR,
+    is_ported BOOLEAN,
+    last_ported DATETIME,
+    caller_name VARCHAR,
+    caller_type VARCHAR,
+    CONSTRAINT phonenumber_unique UNIQUE (value)
+);

migrations/2018-12-23-230955_reverse-dns/down.sql

@@ -0,0 +1,27 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ipaddrs_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    asn INTEGER,
+    as_org VARCHAR,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO _ipaddrs_new (id, family, value, unscoped, continent, continent_code, city, latitude, longitude, asn, as_org)
+  SELECT id, family, value, unscoped, continent, continent_code, city, latitude, longitude, asn, as_org
+  FROM ipaddrs;
+
+DROP TABLE ipaddrs;
+ALTER TABLE _ipaddrs_new RENAME TO ipaddrs;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-23-230955_reverse-dns/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ipaddrs ADD COLUMN description VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN reverse_dns VARCHAR;

migrations/2018-12-24-141533_networks/down.sql

@@ -0,0 +1,3 @@
+DROP TABLE network_devices;
+DROP TABLE networks;
+DROP TABLE devices;

migrations/2018-12-24-141533_networks/up.sql

@@ -0,0 +1,30 @@
+CREATE TABLE networks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+CREATE TABLE devices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    name VARCHAR,
+    hostname VARCHAR,
+    vendor VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    last_seen DATETIME,
+    CONSTRAINT device_unique UNIQUE (value)
+);
+
+CREATE TABLE network_devices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    network_id INTEGER NOT NULL,
+    device_id INTEGER NOT NULL,
+    ipaddr VARCHAR,
+    last_seen DATETIME,
+    FOREIGN KEY(network_id) REFERENCES networks(id) ON DELETE CASCADE,
+    FOREIGN KEY(device_id) REFERENCES devices(id) ON DELETE CASCADE,
+    CONSTRAINT network_device_unique UNIQUE (network_id, device_id)
+);

migrations/2019-01-20-000235_ttl/down.sql

@@ -0,0 +1 @@
+DROP TABLE ttls;

migrations/2019-01-20-000235_ttl/up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE ttls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);

migrations/2019-02-03-123424_accounts/down.sql

@@ -0,0 +1 @@
+DROP TABLE accounts;

migrations/2019-02-03-123424_accounts/up.sql

@@ -0,0 +1,12 @@
+CREATE TABLE accounts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);

migrations/2019-02-11-011316_breaches/down.sql

@@ -0,0 +1,2 @@
+DROP TABLE breach_emails;
+DROP TABLE breaches;

migrations/2019-02-11-011316_breaches/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE breaches (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT breach_unique UNIQUE (value)
+);
+
+CREATE TABLE breach_emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    breach_id INTEGER NOT NULL,
+    email_id INTEGER NOT NULL,
+    password VARCHAR,
+    FOREIGN KEY(breach_id) REFERENCES breaches(id) ON DELETE CASCADE,
+    FOREIGN KEY(email_id) REFERENCES emails(id) ON DELETE CASCADE,
+    CONSTRAINT breach_emails_unique UNIQUE (breach_id, email_id, password)
+);

migrations/2019-03-08-060037_images/down.sql

@@ -0,0 +1 @@
+DROP TABLE images;

migrations/2019-03-08-060037_images/up.sql

@@ -0,0 +1,21 @@
+CREATE TABLE images (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+
+    filename VARCHAR,
+    mime VARCHAR,
+    width INT,
+    height INT,
+    created DATETIME,
+
+    latitude FLOAT,
+    longitude FLOAT,
+
+    nudity FLOAT,
+    ahash VARCHAR,
+    dhash VARCHAR,
+    phash VARCHAR,
+
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT image_unique UNIQUE (value)
+);

migrations/2019-03-24-195306_email-names/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _emails_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO _emails_new (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM emails;
+
+DROP TABLE emails;
+ALTER TABLE _emails_new RENAME TO emails;
+
+PRAGMA foreign_keys=on;

migrations/2019-03-24-195306_email-names/up.sql

@@ -0,0 +1,2 @@
+-- Your SQL goes here
+ALTER TABLE emails ADD COLUMN displayname VARCHAR;

migrations/2019-05-30-142142_ports/down.sql

@@ -0,0 +1 @@
+DROP TABLE ports;

migrations/2019-05-30-142142_ports/up.sql

@@ -0,0 +1,17 @@
+CREATE TABLE ports (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);

migrations/2019-07-04-233515_autonoscope/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE autonoscope;

migrations/2019-07-04-233515_autonoscope/up.sql

@@ -0,0 +1,8 @@
+-- Your SQL goes here
+CREATE TABLE autonoscope (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    object VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    scoped BOOLEAN NOT NULL,
+    CONSTRAINT autonoscope_unique UNIQUE (object, value)
+);

migrations/2019-07-27-152215_netblocks/down.sql

@@ -0,0 +1 @@
+DROP TABLE netblocks;

migrations/2019-07-27-152215_netblocks/up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE netblocks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    asn INTEGER,
+    as_org VARCHAR,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2019-08-11-073709_misc-fields/down.sql

@@ -0,0 +1,41 @@
+PRAGMA foreign_keys=off;
+
+-- accounts
+CREATE TABLE _accounts_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);
+
+INSERT INTO _accounts_new (id, value, service, username, displayname, email, url, last_seen, unscoped)
+    SELECT id, value, service, username, displayname, email, url, last_seen, unscoped
+    FROM accounts;
+
+DROP TABLE accounts;
+ALTER TABLE _accounts_new RENAME TO accounts;
+
+-- networks
+CREATE TABLE _networks_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+INSERT INTO _networks_new (id, value, unscoped, latitude, longitude)
+    SELECT id, value, unscoped, latitude, longitude
+    FROM networks;
+
+DROP TABLE networks;
+ALTER TABLE _networks_new RENAME TO networks;
+
+PRAGMA foreign_keys=on;

migrations/2019-08-11-073709_misc-fields/up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE accounts ADD COLUMN phonenumber VARCHAR;
+ALTER TABLE accounts ADD COLUMN profile_pic VARCHAR;
+ALTER TABLE accounts ADD COLUMN birthday VARCHAR;
+ALTER TABLE networks ADD COLUMN description VARCHAR;

migrations/2019-11-19-154056_cryptoaddr/down.sql

@@ -0,0 +1 @@
+DROP TABLE cryptoaddrs;

migrations/2019-11-19-154056_cryptoaddr/up.sql

@@ -0,0 +1,13 @@
+CREATE TABLE cryptoaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    currency VARCHAR,
+    denominator INTEGER,
+    balance BIGINT,
+    received BIGINT,
+    first_seen DATETIME,
+    last_withdrawal DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2020-01-09-024234_activity/down.sql

@@ -0,0 +1,31 @@
+DROP TABLE activity;
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-01-09-024234_activity/up.sql

@@ -0,0 +1,44 @@
+CREATE TABLE activity (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    topic VARCHAR NOT NULL,
+    time DATETIME NOT NULL,
+    uniq VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    radius INTEGER,
+    content VARCHAR NOT NULL
+);
+CREATE UNIQUE INDEX activity_uniq ON activity(topic, uniq);
+CREATE INDEX activity_topic ON activity(topic);
+CREATE INDEX activity_time ON activity(time);
+CREATE INDEX activity_topic_time ON activity(topic, time);
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, expire)
+    SELECT id, family, key, expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/up.sql

@@ -0,0 +1,19 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, value, expire)
+    SELECT id, family, key, "unknown", expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

modules/dev/asn.lua

@@ -1,16 +0,0 @@
--- Description: Run a asn lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = asn_lookup(arg['value'])
-    if last_err() then return end
-
-    if arg['asn'] ~= lookup['asn'] or arg['as_org'] ~= lookup['as_org'] then
-        db_update('ipaddr', arg, {
-            asn=lookup['asn'],
-            as_org=lookup['as_org'],
-        })
-    end
-end

modules/dev/cname-harvest.lua

@@ -1,54 +0,0 @@
--- Description: Query for CNAMES to find subdomains
--- Version: 0.1.0
--- Source: subdomains
--- License: GPL-3.0
-
-function iter(r)
-    if r == nil then
-        return
-    end
-
-    m = regex_find("(.+)\\.$", r)
-    if last_err() then return end
-
-    if m == nil then
-        return
-    end
-    r = m[2]
-
-    domain = psl_domain_from_dns_name(r)
-    if last_err() then return end
-
-    domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=r,
-        })
-        if last_err() then return end
-    end
-end
-
-function run(arg)
-    records = dns(arg['value'], 'CNAME')
-    if last_err() then return end
-
-    if records['success'] == nil then return end
-    records = records['success']
-
-    -- there is a bug in struct -> lua that causes tables to be zero indexed
-    -- this checks if there's something at index 0 but uses index 1 if this is fixed
-    i = 0
-    if records[i] == nil then i = 1 end
-
-    while records[i] ~= nil do
-        r = records[i]['CNAME']
-        iter(r)
-        if last_err() then return end
-        i = i+1
-    end
-end

modules/dev/ctlogs.lua

@@ -1,48 +0,0 @@
--- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://crt.sh/', {
-        query={
-            q='%.' .. arg['value'],
-            output='json'
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    certs = json_decode_stream(resp['text'])
-    if last_err() then return end
-
-    seen = {}
-
-    i = 1
-    while i <= #certs do
-        c = certs[i]
-        -- print(c)
-
-        name = c['name_value']
-
-        if name:find("*.") == 1 then
-            -- ignore wildcard domains
-            seen[name] = 1
-        end
-
-        if seen[name] == nil then
-            -- info(name)
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=name,
-            })
-            seen[name] = 1
-        end
-
-        i = i+1
-    end
-end

modules/dev/dns-resolve.lua

@@ -1,46 +0,0 @@
--- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.1.0
--- Source: subdomains
--- License: GPL-3.0
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    -- update subdomain
-    resolvable = records['success'] ~= nil
-    if arg['resolvable'] ~= resolvable then
-        -- TODO: pass arg to function as well
-        db_update('subdomain', arg, {
-            resolvable=resolvable
-        })
-    end
-
-    if not resolvable then
-        return
-    end
-
-    records = records['success']
-
-    -- there is a bug in struct -> lua that causes tables to be zero indexed
-    -- this checks if there's something at index 0 but uses index 1 if this is fixed
-    i = 0
-    if records[i] == nil then i = 1 end
-
-    while records[i] ~= nil do
-        r = records[i]
-        if r['A'] ~= nil then
-            ipaddr_id = db_add('ipaddr', {
-                family='4',
-                value=r['A'],
-            })
-            if last_err() then return end
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=arg['id'],
-                ip_addr_id=ipaddr_id,
-            })
-        end
-        i = i+1
-    end
-end

modules/dev/geoip.lua

@@ -1,10 +0,0 @@
--- Description: Run a geoip lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = geoip_lookup(arg['value'])
-    if last_err() then return end
-    db_update('ipaddr', arg, lookup)
-end

modules/dev/hackertarget-subdomains.lua

@@ -1,30 +0,0 @@
--- Description: Query hackertarget for subdomains of a domain
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.hackertarget.com/hostsearch/', {
-        query={
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    m = regex_find_all("([^,]+),.+\\n?", resp['text'])
-
-    i = 1
-    while i <= #m do
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=m[i][2]
-        })
-
-        i = i+1
-    end
-end

modules/dev/otx-subdomains.lua

@@ -1,32 +0,0 @@
--- Description: Query alienvault otx passive dns for subdomains of a domain
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    url = 'https://otx.alienvault.com/api/v1/indicators/domain/' .. arg['value'] .. '/passive_dns'
-
-    req = http_request(session, 'GET', url, {})
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['passive_dns']
-
-    i = 0
-    while o[i] do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x['hostname'],
-        })
-
-        i = i+1
-    end
-end

modules/dev/passive-spider.lua

@@ -1,63 +0,0 @@
--- Description: Scrape known http responses for urls
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function entry(parent, href)
-    -- TODO: parse mailto:foo@example.com?subject=asdf
-    -- TODO: parse tel:+4912345
-    -- TODO: allow discovering 3rd-party domains
-    -- TODO: maybe record urls as well
-
-    local psl, parts, url, host
-
-    if href == nil then
-        return
-    end
-
-    url = url_join(parent, href)
-    if last_err() then return clear_err() end
-    if url:match('^https?://') == nil then
-        return
-    end
-
-    parts = url_parse(url)
-    if last_err() then return end
-    host = parts['host']
-    psl = psl_domain_from_dns_name(host)
-
-
-    domain_id = db_select('domain', psl)
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=host,
-        })
-    end
-end
-
-function run(arg)
-    if arg['body'] == nil or #arg['body'] == 0 then
-        return
-    end
-
-    body = utf8_decode(arg['body'])
-    if last_err() then return end
-
-    links = html_select_list(body, 'a')
-    if last_err() then return end
-
-    if #links == 0 then
-        return
-    end
-
-    -- process html links
-    i = 1
-    while i <= #links do
-        href = links[i]['attrs']['href']
-
-        entry(arg['value'], href)
-
-        i = i+1
-    end
-end

modules/dev/pgp-keyserver.lua

@@ -1,58 +0,0 @@
--- Description: Query pgp keyserver for email addresses
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    --lookup_url = 'https://pgp.mit.edu/pks/lookup'
-    lookup_url = 'https://sks-keyservers.net/pks/lookup'
-
-    req = http_request(session, 'GET', lookup_url, {
-        query={
-            search=arg['value'],
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    links = html_select_list(resp['text'], 'a')
-    i = 1
-    while i <= #links do
-        href = links[i]['attrs']['href']
-
-        if href:find('/pks/lookup%?op=get&search=') == 1 then
-            url = url_join(lookup_url, href)
-
-            req = http_request(session, 'GET', url, {})
-
-            resp = http_send(req)
-            -- TODO: do not abort script if one attempt fails
-            if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-            pubkey = pgp_pubkey_armored(resp['text'])
-
-            -- print(pubkey)
-
-            -- TODO: ensure at least one email matches our target domain
-            if pubkey['uids'] then
-                j = 1
-                while j <= #pubkey['uids'] do
-                    m = regex_find("<([^< ]+@[^< ]+)>$", pubkey['uids'][j])
-                    if m then
-                        db_add('email', {
-                            value=m[2],
-                        })
-                    end
-                    j = j+1
-                end
-            end
-        end
-
-        i = i+1
-    end
-end

modules/dev/threatminer-ipaddr.lua

@@ -1,52 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- TODO: add option to filter old entries based on last_seen
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/host.php', {
-        query={
-            rt='2',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    i = 0
-    while o[i] do
-        x = o[i]
-
-        domain = psl_domain_from_dns_name(x['domain'])
-        -- TODO: if this fails, skip this entry instead
-        if last_err() then return end
-
-        domain_id = db_add('domain', {
-            value=domain,
-        })
-
-        if domain_id ~= nil then
-            subdomain_id = db_add('subdomain', {
-                domain_id=domain_id,
-                value=x['domain'],
-            })
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=subdomain_id,
-                ip_addr_id=arg['id'],
-            })
-        end
-
-        i = i+1
-    end
-end

modules/dev/threatminer-subdomains.lua

@@ -1,35 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of a domain
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/domain.php', {
-        query={
-            rt='5',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    i = 0
-    while o[i] do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x,
-        })
-
-        i = i+1
-    end
-end

modules/dev/url-scan.lua

@@ -1,46 +0,0 @@
--- Description: Scan subdomains for websites
--- Version: 0.1.0
--- Source: subdomains
--- License: GPL-3.0
-
-function request(subdomain_id, url)
-    req = http_request(session, 'GET', url, {
-        timeout=5000
-    })
-    reply = http_send(req)
-
-    if last_err() then
-        clear_err()
-        return
-    end
-
-    obj = {
-        subdomain_id=subdomain_id,
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-        redirect=reply['headers']['location'],
-    }
-
-    redirect = reply['headers']['location']
-    if redirect then
-        obj['redirect'] = url_join(url, redirect)
-    end
-
-    db_add('url', obj)
-
-    -- info(json_encode(reply['status']))
-    -- info(json_encode(reply['headers']['location']))
-    -- info(json_encode(reply['text']))
-end
-
-function run(arg)
-    domain = arg['value']
-
-    session = http_mksession()
-
-    request(arg['id'], 'http://' .. domain .. '/')
-    if last_err() then return end
-    request(arg['id'], 'https://' .. domain .. '/')
-    if last_err() then return end
-end

modules/dev/waybackurls.lua

@@ -1,56 +0,0 @@
--- Description: Discover subdomains from wayback machine
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-
-    domain = arg['value']
-    url = 'https://web.archive.org/cdx/search/cdx?url=*.' .. domain .. '/*&output=json&collapse=urlkey'
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-
-    -- no known urls
-    if o[0] == nil then
-        return
-    end
-
-    -- ensure the api response is still what we expect
-    if o[0][2] == nil then
-        return 'api returned unexpected json format'
-    end
-
-    seen = {}
-
-    i = 1
-    while o[i] do
-        url = o[i][2]
-
-        parts = url_parse(url)
-
-        if last_err() then
-            clear_err()
-            error("Failed to parse url: " .. json_encode(url))
-        else
-            subdomain = parts['host']
-
-            if seen[subdomain] == nil then
-                db_add('subdomain', {
-                    domain_id=arg['id'],
-                    value=parts['host'],
-                })
-                if last_err() then return end
-                seen[subdomain] = 1
-            end
-        end
-
-        i = i+1
-    end
-end

modules/harness/activity-ping-once.lua

@@ -0,0 +1,26 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    db_activity({
+        topic=topic,
+        time=sn0int_time(),
+        uniq=uniq,
+        latitude=lat,
+        longitude=lon,
+        radius=radius,
+        content={
+            msg='ohai',
+        },
+    })
+end

modules/harness/activity-ping.lua

@@ -0,0 +1,29 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    while true do
+        db_activity({
+            topic=topic,
+            time=sn0int_time(),
+            uniq=uniq,
+            latitude=lat,
+            longitude=lon,
+            radius=radius,
+            content={
+                msg='ohai',
+            },
+        })
+        sleep(5)
+    end
+end

modules/harness/add-all.lua

@@ -0,0 +1,107 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('adding domain')
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+    if last_err() then return end
+
+    info('adding subdomain')
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value='example.com',
+    })
+    if last_err() then return end
+
+    info('adding ipaddr')
+    ipaddr_id = db_add('ipaddr', {
+        value='192.0.2.1',
+    })
+    if last_err() then return end
+
+    info('adding device')
+    device_id = db_add('device', {
+        value='ff:ff:ff:ff:ff:ff',
+    })
+    if last_err() then return end
+
+    info('adding network')
+    network_id = db_add('network', {
+        value='myssid',
+    })
+    if last_err() then return end
+
+    info('adding email')
+    email_id = db_add('email', {
+        value='foo@example.com',
+    })
+    if last_err() then return end
+
+    info('adding phonenumber')
+    phonenumber_id = db_add('phonenumber', {
+        value='+4912345678',
+    })
+    if last_err() then return end
+
+    info('adding breach')
+    breach_id = db_add('breach', {
+        value='hack the planet',
+    })
+    if last_err() then return end
+
+    info('adding account')
+    account_id = db_add('account', {
+        service='github.com',
+        username='kpcyrd',
+    })
+    if last_err() then return end
+
+    info('adding image')
+    blob = create_blob('abc')
+    image_id = db_add('image', {
+        value=blob,
+    })
+    if last_err() then return end
+
+    info('adding port')
+    port_id = db_add('port', {
+        ip_addr_id=ipaddr_id,
+        ip_addr='192.0.2.1',
+        port=443,
+        protocol='tcp',
+        status='open',
+    })
+    if last_err() then return end
+
+    info('adding url')
+    url_id = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://www.example.com/a/b',
+        body='<html></html>',
+    })
+    if last_err() then return end
+
+    info('adding breach_email')
+    db_add('breach-email', {
+        breach_id=breach_id,
+        email_id=email_id,
+    })
+    if last_err() then return end
+
+    info('adding network_device')
+    db_add('network-device', {
+        network_id=network_id,
+        device_id=device_id,
+    })
+    if last_err() then return end
+
+    info('adding subdomain_ipaddr')
+    db_add('subdomain-ipaddr', {
+        subdomain_id=subdomain_id,
+        ip_addr_id=ipaddr_id,
+    })
+    if last_err() then return end
+end

modules/harness/commit-log.lua

@@ -0,0 +1,26 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- git log -s --format='%H %ci'
+
+function run()
+    while true do
+        local x = stdin_readline()
+        if x == nil then
+            break
+        end
+        local m = regex_find('^(\\S+) (.+)', x)
+        if m then
+            time = strptime('%Y-%m-%d %T %z', m[3])
+            time = sn0int_time_from(time)
+
+            db_activity({
+                topic='harness/sn0int-commit:dummy',
+                time=time,
+                uniq=m[2],
+                content={},
+            })
+        end
+    end
+end

modules/harness/dummy-resolve.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: subdomains
+
+function run(arg)
+    db_update('subdomain', arg, {
+        resolvable=true,
+    })
+end

modules/harness/dummy-subdomains.lua

@@ -0,0 +1,13 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    for i=1, 20 do
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=http_mksession() .. '.' .. arg['value'],
+        })
+    end
+end

modules/harness/fail.lua

@@ -1,6 +1,5 @@
 -- Description: Test error handling
 -- Version: 0.1.0
--- Source: domains
 -- License: GPL-3.0
 
 function run()

modules/harness/geo-polygon-contains.lua

@@ -0,0 +1,74 @@
+-- Description: demonstrate geofencing with polygons
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    hamburg = {
+        { lat=53.63975308945899,  lon=9.764785766601562  },
+        { lat=53.59494998253459,  lon=9.827270507812     },
+        { lat=53.663153974456456, lon=9.9151611328125    },
+        { lat=53.65582987649682,  lon=9.976272583007812  },
+        { lat=53.68613523817129,  lon=9.992752075195312  },
+        { lat=53.68674518938816,  lon=10.051460266113281 },
+        { lat=53.72495117617815,  lon=10.075492858886719 },
+        { lat=53.71946627930625,  lon=10.118408203125    },
+        { lat=53.743635083157756, lon=10.164413452148438 },
+        { lat=53.73104466704585,  lon=10.202865600585938 },
+        { lat=53.676781546441546, lon=10.16304016113281  },
+        { lat=53.632832079199474, lon=10.235824584960938 },
+        { lat=53.608803292930894, lon=10.2008056640625   },
+        { lat=53.578646152866504, lon=10.208358764648438 },
+        { lat=53.57212285981298,  lon=10.163726806640625 },
+        { lat=53.52071674896369,  lon=10.18707275390625  },
+        { lat=53.52643162253097,  lon=10.224151611328125 },
+        { lat=53.44062753992289,  lon=10.347747802734375 },
+        { lat=53.38824275010831,  lon=10.248870849609375 },
+        { lat=53.38824275010831,  lon=10.15960693359375  },
+        { lat=53.44635321212876,  lon=10.064849853515625 },
+        { lat=53.40595029739904,  lon=9.985198974609375  },
+        { lat=53.42385506057106,  lon=9.951210021972656  },
+        { lat=53.41843327091211,  lon=9.944171905517578  },
+        { lat=53.41812635648326,  lon=9.927349090576172  },
+        { lat=53.412294561442884, lon=9.917736053466797  },
+        { lat=53.41464783813818,  lon=9.901256561279297  },
+        { lat=53.443490472483326, lon=9.912586212158201  },
+        { lat=53.45177144115704,  lon=9.897651672363281  },
+        { lat=53.43633277935392,  lon=9.866924285888672  },
+        { lat=53.427639673754776, lon=9.866409301757812  },
+        { lat=53.427639673754776, lon=9.858856201171875  },
+        { lat=53.46710230573499,  lon=9.795513153076172  },
+        { lat=53.49039461941655,  lon=9.795341491699219  },
+        { lat=53.49029248806277,  lon=9.77903366088867   },
+        { lat=53.49856433088649,  lon=9.780235290527344  },
+        { lat=53.5078554643033,   lon=9.758434295654297  },
+        { lat=53.545407634092975, lon=9.759807586669922  },
+        { lat=53.568147234570084, lon=9.633293151855469  },
+        { lat=53.58802162343514,  lon=9.655780792236328  },
+        { lat=53.568351121879815, lon=9.727706909179688  },
+        { lat=53.60921067445695,  lon=9.737663269042969  },
+    }
+
+    points = {
+        {
+            name='Alice',
+            lat=52.52437,
+            lon=13.41053,
+        }, {
+            name='Bob',
+            lat=53.551085,
+            lon=9.993682,
+        }, {
+            name='Charlie',
+            lat=40.726662,
+            lon=-74.036677,
+        }
+    }
+
+    for i=1, #points do
+        if geo_polygon_contains(hamburg, points[i]) then
+            info('[INSIDE ] ' .. points[i]['name'])
+        else
+            info('[OUTSIDE] ' .. points[i]['name'])
+        end
+    end
+end

modules/harness/google-tls.lua

@@ -0,0 +1,29 @@
+-- Description: Test various tls functions
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('sending https request to google.com')
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://google.com/', {})
+    r = http_send(req)
+    if last_err() then return end
+    debug(r)
+
+    info('creating tls socket to google.com')
+    sock = sock_connect('google.com', 443, {
+        tls=true,
+    })
+    if last_err() then return end
+    debug(sock)
+
+    info('creating socket to google.com, wrapping afterwards')
+    sock = sock_connect('google.com', 443, {})
+    if last_err() then return end
+    tls = sock_upgrade_tls(sock, {
+        sni_value='google.com',
+    })
+    if last_err() then return end
+    debug(sock)
+    debug(tls)
+end

modules/harness/import-subdomains.lua

@@ -0,0 +1,29 @@
+-- Description: Import subdomains from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lu
+
+    while true do
+        local line = stdin_readline()
+        if line == nil then
+            break
+        end
+        -- strip newline
+        local m = regex_find('.+', line)
+        if m then
+            local subdomain = m[1]
+
+            local domain = psl_domain_from_dns_name(subdomain)
+            local domain_id = db_add('domain', {
+                value=domain,
+            })
+
+            db_add('subdomain', {
+                domain_id=domain_id,
+                value=subdomain,
+            })
+        end
+    end
+end