Release v0.11.1

Fix build for openbsd

.dockerignore

@@ -1,6 +1,9 @@
 target
 Dockerfile
 .dockerignore
+docker
+docs
+ci
 .git
 .gitignore
 *.sw[op]

.travis.yml

@@ -1,6 +1,7 @@
 language: rust
 
-cache: cargo
+# upload takes too long on windows and gets killed due to inactivity
+#cache: cargo
 
 matrix:
   include:

CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# How to contribute
+
+To contribute to sn0int, clone the repository and make sure both the build and
+tests pass for you:
+
+    git clone https://github.com/kpcyrd/sn0int.git
+    cd sn0int
+    # build the project
+    cargo build
+    # run regular tests
+    cargo test
+    # run tests depending on the network
+    # these might fail if a service is down
+    cargo test -- --ignored
+
+The project is loosely structured into a few folders:
+
+- `src/models/` - database models
+- `src/runtime/` - the stdlib that's exposed to lua
+- `src/engine/` - code related to lua
+- `src/sandbox/` - code related to sandboxing
+- `src/cmd/` - cli commands
+- `src/` - misc modules
+
+After you're done, make sure the build completes without any warnings and both
+tests pass successfully:
+
+    cargo test
+    cargo test -- --ignored
+
+If you want to introduce a new feature feel free to open an issue first to make
+sure your feature is a good fit for the project before implementing it.

Cargo.toml

@@ -1,23 +1,34 @@
 [package]
 name = "sn0int"
-version = "0.5.1"
-description = "OSINT framework and package manager"
+version = "0.11.1"
+description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
+edition = "2018"
+
+[profile.release]
+# skip lto to avoid compiler bug:
+# https://github.com/kpcyrd/sn0int/issues/77
+# https://github.com/rust-lang/rust/issues/58674
+opt-level = 1
+lto = false
 
 [badges]
 travis-ci = { repository = "kpcyrd/sn0int" }
 
 [workspace]
-members = ["sn0int-registry/sn0int-common",
-           "sn0int-registry"]
+# rocket is broken and blocks the whole workspace, removing
+#members = ["sn0int-registry/sn0int-common",
+#           "sn0int-registry"]
+members = ["sn0int-registry/sn0int-common"]
+exclude = ["sn0int-registry"]
 
 [dependencies]
-sn0int-common = { version="0.3.0", path="sn0int-registry/sn0int-common" }
-rustyline = "2"
+sn0int-common = { version="0.6.0", path="sn0int-registry/sn0int-common" }
+rustyline = "4.0"
 log = "0.4"
 env_logger = "0.6"
 hlua-badtouch = "0.4"
@@ -28,12 +39,13 @@ colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
 publicsuffix = { version="1.5", default-features=false }
-diesel = { version = "1.0.0", features = ["sqlite"] }
+diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
+chrono = { version = "0.4", features = ["serde"] }
 dirs = "1.0"
 url = "1.7"
-chrootable-https = "0.3.4"
-trust-dns-proto = "0.5.0"
+#chrootable-https = { path = "../chrootable-https" }
+chrootable-https = "0.9"
 base64 = "0.10"
 kuchiki = "0.7.2"
 serde_urlencoded = "0.5"
@@ -43,27 +55,52 @@ serde_json = "1.0"
 crossbeam-channel = "0.3"
 ctrlc = "3.1"
 opener = "0.3.0"
-separator = "0.3.1"
+separator = "0.4"
 maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.2"
 regex = "1.0"
-toml = "0.4"
-maxminddb = "0.10.0"
+toml = "0.5"
+maxminddb = "0.13"
 tar = "0.4.17"
 libflate = "0.1.14"
 threadpool = "1.7"
 x509-parser = "0.4.0"
 der-parser = "1.1.0"
 nom = "4.1.1"
+atty = "0.2"
+bufstream = "0.1.4"
+tokio = "0.1.14"
+semver = "0.9"
+bytes = "0.4"
+
+digest = "0.8.0"
+hex = "0.3.1"
+bs58 = "0.2.2"
+blake2 = "0.8.0"
+md-5 = "0.8.0"
+sha-1 = "0.8.1"
+sha2 = "0.8.0"
+
+image = "0.21"
+kamadak-exif = "0.3.1"
+walkdir = "2.2"
+nude = "0.1.0"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
-syscallz = "0.7"
-nix = "0.11"
+#syscallz = { path="../syscallz-rs" }
+syscallz = "0.11"
+nix = "0.13"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.3.1"
 unveil = "0.2.0"
 
 [dev-dependencies]
-boxxy = "0.8"
+#boxxy = { path = "../boxxy-rs" }
+boxxy = "0.10"
+tempfile = "3.0"
+
+#[patch.crates-io]
+#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }
+#rocket_contrib = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }

Dockerfile

@@ -1,13 +1,14 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM debian
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

ISSUE_TEMPLATE.md

@@ -0,0 +1,29 @@
+<!--
+Hello!
+
+If you want to report a bug we added some common questions below that help us analyse your issue faster.
+
+All of these are optional so feel free to remove anything that doesn't apply.
+-->
+
+please describe your issue here
+
+---
+
+## Versions
+
+- **rustc --version:**
+- **cargo --version:**
+- **sn0int --version:**
+- **uname -a:**
+
+## Environment
+
+- **Operating System/Distro:**
+- **Installed from (source/apt/pacman/brew/docker):**
+
+<!--
+Thank you!
+
+We'll try to respond as quickly as possible.
+-->

Makefile

@@ -14,3 +14,15 @@ test:
 	(cd sn0int-registry; cargo test)
 	cargo test
 	cargo test -- --ignored
+
+update:
+	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt
+	get-iab -v -u http://standards-oui.ieee.org/iab/iab.txt -f data/ieee-iab.txt
+	rm -f data/ieee-*.txt.bak
+
+docs:
+	$(MAKE) -C docs html
+	contrib/html-toc2md.pl README.md docs/_build/html/index.html > README2.md
+	mv README2.md README.md
+
+.PHONY: check force-check test update docs

README.md

@@ -1,4 +1,4 @@
-# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs]
+# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
 [travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
 [travis]:       https://travis-ci.org/kpcyrd/sn0int
@@ -6,39 +6,188 @@
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
 [docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
+[irc-img]:      https://img.shields.io/badge/hackint-%23sn0int-blue.svg
+[irc]:          https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int
+[mastodon-img]: https://img.shields.io/badge/mastodon-chaos.social-blue.svg
+[mastodon]:     https://chaos.social/@sn0int
+[registry-img]: https://img.shields.io/website/https/sn0int.com.svg?label=registry
+[registry]:     https://sn0int.com/
 
-sn0int is an OSINT framework and package manager. It was built for IT security
-professionals and bug hunters to gather intelligence about a given target or
-about yourself. sn0int is enumerating attack surface by semi-automatically
-processing public information and mapping the results in a unified format for
-followup investigations.
+sn0int is a semi-automatic OSINT framework and package manager. It was built
+for IT security professionals and bug hunters to gather intelligence about a
+given target or about yourself. sn0int is enumerating attack surface by
+semi-automatically processing public information and mapping the results in a
+unified format for followup investigations.
 
 Among other things, sn0int is currently able to:
 
-- [X] Harvest subdomains from certificate transparency logs
-- [X] Harvest subdomains from various passive dns logs
-- [X] Sift through subdomain results for publicly accessible websites
-- [X] Harvest emails from pgp keyservers
-- [X] Enrich ip addresses with ASN and geoip info
-- [X] Harvest subdomains from the wayback machine
+- Harvest subdomains from certificate transparency logs and passive dns
+- Enrich ip addresses with asn and geoip info
+- Harvest emails from pgp keyservers and whois
+- Discover compromised logins in breaches
+- Find somebody's profiles across the internet
+- Enumerate local networks with unique techniques like passive arp
+- Gather information about phonenumbers
+- Harvest data and images from instagram profiles
+- Scan images for nudity
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
-and is fully opensource.  None of the investigations listed above are hardcoded
+and is fully opensource. None of the investigations listed above are hardcoded
 in the source, instead those are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
 you to ship updates for your modules on your own since you don't need to send a
 pull request.
 
-Join us on IRC: <ircs://irc.hackint.org/#sn0int>
+For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
+
+[![asciicast](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB.svg)](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB)
 
 ## Installation
 
-- Archlinux: `yaourt -S sn0int`
-- Alpine: `apk add --no-cache sqlite-dev libseccomp-dev cargo` + build from source
-- Debian: `apt install libsqlite3-dev libseccomp-dev` + build from source
-- OpenBSD: `pkg_add sqlite3` + build from source
-- OSX: `brew install sqlite3` + build from source
+Archlinux
+
+    pacman -S sn0int
+
+Mac OSX
+
+    brew install sn0int
+
+For everything else please have a look at the [detailed list][1].
+
+[1]: https://sn0int.readthedocs.io/en/latest/install.html
+
+## Getting started
+
+- [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
+  - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
+  - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
+  - [Debian testing/Debian sid/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-testing-debian-sid-kali)
+  - [Ubuntu/Debian stable](https://sn0int.readthedocs.io/en/latest/install.html#ubuntu-debian-stable)
+  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
+  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
+  - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
+- [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
+  - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
+  - [Adding something to scope](https://sn0int.readthedocs.io/en/latest/usage.html#adding-something-to-scope)
+  - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
+  - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
+  - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
+- [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
+  - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
+  - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
+- [Database](https://sn0int.readthedocs.io/en/latest/database.html)
+  - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
+  - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
+- [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/structs.html#domains)
+  - [Subdomains](https://sn0int.readthedocs.io/en/latest/structs.html#subdomains)
+  - [IpAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#ipaddrs)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/structs.html#urls)
+  - [Emails](https://sn0int.readthedocs.io/en/latest/structs.html#emails)
+  - [Phonenumbers](https://sn0int.readthedocs.io/en/latest/structs.html#phonenumbers)
+  - [Devices](https://sn0int.readthedocs.io/en/latest/structs.html#devices)
+  - [Networks](https://sn0int.readthedocs.io/en/latest/structs.html#networks)
+  - [Accounts](https://sn0int.readthedocs.io/en/latest/structs.html#accounts)
+  - [Breaches](https://sn0int.readthedocs.io/en/latest/structs.html#breaches)
+  - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
+  - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
+- [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
+- [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
+- [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
+  - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
+  - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
+  - [Using access keys as source argument](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-as-source-argument)
+- [Configuration](https://sn0int.readthedocs.io/en/latest/config.html)
+  - [\[core\]](https://sn0int.readthedocs.io/en/latest/config.html#core)
+  - [\[namespaces\]](https://sn0int.readthedocs.io/en/latest/config.html#namespaces)
+  - [\[network\]](https://sn0int.readthedocs.io/en/latest/config.html#network)
+- [Sandbox](https://sn0int.readthedocs.io/en/latest/sandbox.html)
+  - [Linux](https://sn0int.readthedocs.io/en/latest/sandbox.html#linux)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/sandbox.html#openbsd)
+  - [IPC Protocol](https://sn0int.readthedocs.io/en/latest/sandbox.html#ipc-protocol)
+  - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
+  - [Diagnosing a sandbox failure](https://sn0int.readthedocs.io/en/latest/sandbox.html#diagnosing-a-sandbox-failure)
+- [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
+  - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
+  - [create_blob](https://sn0int.readthedocs.io/en/latest/reference.html#create-blob)
+  - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
+  - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
+  - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
+  - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
+  - [error](https://sn0int.readthedocs.io/en/latest/reference.html#error)
+  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
+  - [geoip_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#geoip-lookup)
+  - [html_select](https://sn0int.readthedocs.io/en/latest/reference.html#html-select)
+  - [html_select_list](https://sn0int.readthedocs.io/en/latest/reference.html#html-select-list)
+  - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
+  - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
+  - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
+  - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
+  - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
+  - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
+  - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
+  - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
+  - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
+  - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
+  - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
+  - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
+  - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [sha1](https://sn0int.readthedocs.io/en/latest/reference.html#sha1)
+  - [sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-256)
+  - [sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-512)
+  - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
+  - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
+  - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
+  - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
+  - [sock_recvline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline)
+  - [sock_recvall](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvall)
+  - [sock_recvline_contains](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline-contains)
+  - [sock_recvline_regex](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline-regex)
+  - [sock_recvn](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvn)
+  - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
+  - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
+  - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
+  - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
+  - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
+  - [url_escape](https://sn0int.readthedocs.io/en/latest/reference.html#url-escape)
+  - [url_join](https://sn0int.readthedocs.io/en/latest/reference.html#url-join)
+  - [url_parse](https://sn0int.readthedocs.io/en/latest/reference.html#url-parse)
+  - [url_unescape](https://sn0int.readthedocs.io/en/latest/reference.html#url-unescape)
+  - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
+  - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
+
+## Rationale
+
+This tool was written for companies to help them understand their attack
+surface from a blackbox point of view. It's often difficult to understand that
+something is easier to discover than some people assume, putting them at risk
+of false security.
+
+It's also designed to be useful for red team assessments and bug bounties,
+which also help companies to identify weaknesses that could result in a
+compromise.
+
+Some functionality was written to do the same thing for individuals to raise
+awareness about personal attack surface, privacy and how much data is publicly
+available. These issues are often out of scope in bug bounties and sometimes by
+design. We believe that blaming the user is the wrong approach and these issues
+should be addressed at the root cause by the people designing those systems.
 
 ## License
 

ci/boxxy_stage1.txt

@@ -1,3 +1,5 @@
 stage1
 ls
+echo checkpoint1
+id
 echo sandbox fail

ci/integration.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+import subprocess
+from subprocess import DEVNULL, PIPE
+import tempfile
+import json
+import sys
+
+
+def _sn0int(tempdir, binary, args, piped_stdout=False):
+    return subprocess.Popen(
+        ['/usr/bin/env', 'HOME='+tempdir, binary] + args,
+        stdin=PIPE,
+        stdout=PIPE if piped_stdout else None,
+    )
+
+
+def sn0int(tempdir, binary, cmds):
+    p = _sn0int(tempdir, binary, [])
+    for cmd in cmds:
+        p.stdin.write((cmd + '\n').encode('utf-8'))
+    p.communicate()
+    if p.returncode != 0:
+        raise Exception('process failed')
+
+
+def sn0int_select(tempdir, binary, query):
+    p = _sn0int(tempdir, binary, ['select', '--json'] + query, piped_stdout=True)
+    stdout, _ = p.communicate()
+    lines = filter(None, stdout.decode('utf-8').split('\n'))
+    return [json.loads(x) for x in lines]
+
+
+def main(tempdir, binary):
+    print('[*] setting up workspace')
+    sn0int(tempdir, binary, [])
+
+    print('[*] adding domain')
+    sn0int(tempdir, binary, [
+        'add domain',
+        'example.com',
+        'select domains',
+    ])
+
+    print('[*] testing db for domain')
+    domains = sn0int_select(tempdir, binary, ['domains'])
+    assert domains == [{'id': 1, 'value': 'example.com', 'unscoped': False}]
+
+    print('[*] installing modules')
+    sn0int(tempdir, binary, [
+        'mod install kpcyrd/ctlogs',
+        'mod install kpcyrd/dns-resolve',
+        'mod install kpcyrd/url-scan',
+        'mod install kpcyrd/geoip',
+    ])
+
+    print('[*] running ctlogs')
+    sn0int(tempdir, binary, [
+        'use ctlogs',
+        'run',
+        'select subdomains',
+    ])
+
+    print('[*] testing db for subdomains')
+    subdomains = sn0int_select(tempdir, binary, ['subdomains'])
+    assert {x['value'] for x in subdomains} == {
+        'www.example.com',
+        'm.example.com',
+        'dev.example.com',
+        'products.example.com',
+        'support.example.com',
+    }
+
+    print('[*] running dns-resolve')
+    sn0int(tempdir, binary, [
+        'use dns-resolve',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs')
+    ipaddrs = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert len(ipaddrs) >= 1
+
+    print('[*] running url-scan')
+    sn0int(tempdir, binary, [
+        'use url-scan',
+        'run',
+        'select urls',
+    ])
+
+    print('[*] testing db for urls')
+    urls = sn0int_select(tempdir, binary, ['urls'])
+    assert {(x['value'], x['status']) for x in urls} == {
+        ('http://www.example.com/', 200),
+        ('https://www.example.com/', 200),
+    }
+
+    print('[*] running geoip')
+    sn0int(tempdir, binary, [
+        'use geoip',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs again')
+    ipaddrs2 = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert ipaddrs != ipaddrs2
+
+    print('')
+    print('\t###########')
+    print('\t# SUCCESS #')
+    print('\t###########')
+    print('')
+
+
+if __name__ == '__main__':
+    try:
+        binary = sys.argv[1]
+    except IndexError:
+        print('Usage: %s target/release/sn0int' % sys.argv[0])
+    else:
+        with tempfile.TemporaryDirectory(prefix='sn0int-') as tempdir:
+            main(tempdir, binary)

ci/run.sh

@@ -9,8 +9,8 @@ case "$1" in
         ci/run.sh build
         wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
              https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind-dl -- -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind-dl -- -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
+        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
+        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
         cargo test --verbose
         cargo test --verbose -- --ignored
         ;;

ci/setup.sh

@@ -5,9 +5,6 @@ case "$1" in
         sudo apt update
         sudo apt install libsqlite3-dev libseccomp-dev
         ;;
-    osx)
-        brew install sqlite3
-        ;;
     windows)
         curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
         7z e sqlite3.zip -y

contrib/docker/Dockerfile.alpine

@@ -0,0 +1,15 @@
+FROM alpine:edge
+RUN apk add --no-cache sqlite-dev libseccomp-dev
+RUN apk add --no-cache --virtual .build-rust rust cargo
+WORKDIR /usr/src/sn0int
+COPY . .
+RUN cargo build --release --verbose
+RUN strip target/release/sn0int
+
+FROM alpine:edge
+RUN apk add --no-cache libgcc sqlite-libs libseccomp
+COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+VOLUME ["/data", "/cache"]
+ENV XDG_DATA_HOME=/data \
+    XDG_CACHE_HOME=/cache
+ENTRYPOINT ["sn0int"]

contrib/docker/Dockerfile.debian

@@ -0,0 +1,16 @@
+FROM rust
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /usr/src/sn0int
+COPY . .
+RUN cargo build --release --verbose
+RUN strip target/release/sn0int
+
+FROM debian
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+VOLUME ["/data", "/cache"]
+ENV XDG_DATA_HOME=/data \
+    XDG_CACHE_HOME=/cache
+ENTRYPOINT ["sn0int"]

contrib/html-toc2md.pl

@@ -0,0 +1,39 @@
+#!/usr/bin/env perl
+use strict; use warnings;
+
+my ($readme, $toc) = @ARGV;
+defined $readme or die 'missing readme path';
+defined $toc or die 'missing toc path';
+
+open(my $r, "<$readme") or die 'failed to open readme';
+open(my $t, "<$toc") or die 'failed to open toc';
+
+my $re = qr/^\s*- \[.+\]\(https:\/\/sn0int.readthedocs.io\/en\/.+\)$/;
+
+# pass through start of readme
+while (<$r>) {
+    last if ($_ =~ $re);
+    print $_;
+}
+
+# skip toc
+while (<$r>) {
+    last unless ($_ =~ $re);
+}
+
+# generate new toc
+while (my $line = <$t>) {
+    if ($line =~ /toctree-l(\d).*href="([^"]+)">(.+)<\/a/) {
+        my $space = $1;
+        my $section = $2;
+        my $label = $3;
+        $label =~ s/([\[\]])/\\$1/g;
+        print $space==2?"  ":"", "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
+    }
+}
+print;
+
+# pass through end of readme
+while (<$r>) {
+    print $_;
+}

docs/conf.py

@@ -138,7 +138,7 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('man', 'sn0int', 'OSINT framework and package manager',
+    ('man', 'sn0int', 'Semi-automatic OSINT framework and package manager',
      [author], 1)
 ]
 

docs/config.rst

@@ -0,0 +1,54 @@
+Configuration
+=============
+
+This section documents the config file. By default this file does not exist and
+a default configuration is used instead.
+
+Linux/BSD
+    ``~/.config/sn0int.toml``
+
+OSX
+    ``~/Library/Preferences/sn0int.toml``
+
+Windows
+    ``%APPDATA%/sn0int.toml``
+
+[core]
+------
+
+``registry``
+    Configure the registry you want to use. Defaults to ``https://sn0int.com``.
+``no-autoupdate``
+    sn0int is going to check if your modules are outdated during startout once
+    a week. Set this option to ``true`` to disable this.
+
+[namespaces]
+------------------
+
+By default sn0int modules are assumed to be installed from the registry. You
+may want to keep a local directory with private modules, especially during
+development. You can configure a folder that contains modules that aren't
+managed by sn0int by adding a namespace section to the config file::
+
+    [namespaces]
+    foo = "/opt/sn0int/foo"
+    bar = "~/repos/a/b/c/sn0int-modules"
+
+This is going to load modules from these two folders and register them in the
+``foo`` and ``bar`` namespace.
+
+Note that sn0int is also going to assume that symlinks in
+``~/.local/share/sn0int/modules`` and folders containing a ``.git`` folder are
+externally managed.
+
+[network]
+---------
+
+To enable a proxy, add the following to your config file::
+
+    [network]
+    proxy = "127.0.0.1:9050"
+
+This forces everything through tor (or any other socks5 proxy) and restricts
+all other functions that depend on the network. For example the ``dns``
+function is fully disabled if a proxy is configured.

docs/index.rst

@@ -1,20 +1,22 @@
 sn0int
 ======
 
-sn0int is an OSINT framework and package manager. It was built for IT security
-professionals and bug hunters to gather intelligence about a given target or
-about yourself. sn0int is enumerating attack surface by semi-automatically
-processing public information and mapping the results in a unified format for
-followup investigations.
+sn0int is a semi-automatic OSINT framework and package manager. It was built
+for IT security professionals and bug hunters to gather intelligence about a
+given target or about yourself. sn0int is enumerating attack surface by
+semi-automatically processing public information and mapping the results in a
+unified format for followup investigations.
 
 Among other things, sn0int is currently able to:
 
-- [X] Harvest subdomains from certificate transparency logs
-- [X] Harvest subdomains from various passive dns logs
-- [X] Sift through subdomain results for publicly accessible websites
-- [X] Harvest emails from pgp keyservers
-- [X] Enrich ip addresses with ASN and geoip info
-- [X] Harvest subdomains from the wayback machine
+- Harvest subdomains from certificate transparency logs
+- Harvest subdomains from various passive dns logs
+- Sift through subdomain results for publicly accessible websites
+- Harvest emails from pgp keyservers
+- Enrich ip addresses with ASN and geoip info
+- Harvest subdomains from the wayback machine
+- Gather information about phonenumbers
+- Bruteforce interesting urls
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource.  None of the investigations listed above are hardcoded
@@ -24,7 +26,7 @@ them with other users by publishing them to the sn0int registry. This allows
 you to ship updates for your modules on your own since you don't need to send a
 pull request.
 
-Join us on IRC: ircs://irc.hackint.org/#sn0int
+Join us on IRC: `irc.hackint.org:6697/#sn0int <https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int>`_
 
 Getting Started
 ---------------
@@ -37,4 +39,8 @@ Getting Started
    usage
    scripting
    database
+   structs
+   keyring
+   config
+   sandbox
    reference

docs/install.rst

@@ -8,17 +8,50 @@ Archlinux
 
 .. code-block:: bash
 
-    $ yaourt -S sn0int
+    $ pacman -S sn0int
 
-Debian/Ubuntu/Kali
-------------------
+Mac OSX
+-------
 
 .. code-block:: bash
 
-    $ apt install libsqlite3-dev libseccomp-dev
+    $ brew install sn0int
+
+Debian testing/Debian sid/Kali
+------------------------------
+
+Note that debian `doesn't ship the geoip2-database
+<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
+download them automatically during the first run.
+
+.. code-block:: bash
+
+    $ apt install build-essential cargo libsqlite3-dev libseccomp-dev publicsuffix
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ cargo install -f
+    $ cargo install -f --path .
+
+Ubuntu/Debian stable
+--------------------
+
+cargo in the repos is too old and the build is `going to fail
+<https://github.com/kpcyrd/sn0int/issues/68>`_. You should either install the
+most recent rust version with `rustup <https://rustup.rs/>`_ or use the docker
+instructions instead.
+
+.. code-block:: bash
+
+    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f --path .
+
+Docker
+------
+
+.. code-block:: bash
+
+    $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
 
 Alpine
 ------
@@ -28,7 +61,7 @@ Alpine
     $ apk add --no-cache sqlite-dev libseccomp-dev cargo
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ cargo install -f
+    $ cargo install -f --path .
 
 OpenBSD
 -------
@@ -38,17 +71,7 @@ OpenBSD
     $ pkg_add sqlite3
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ cargo install -f
-
-Mac OSX
--------
-
-.. code-block:: bash
-
-    $ brew install sqlite3
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f
+    $ cargo install -f --path .
 
 Windows
 -------
@@ -63,4 +86,4 @@ This is not recommended and only passively maintained. Please prefer linux in a
     $ 7z e sqlite3.zip -y
     $ "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
     $ export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
-    $ cargo install -f
+    $ cargo install -f --path .

docs/keyring.rst

@@ -0,0 +1,73 @@
+Keyring
+=======
+
+A common problem is that you need either an api key or a username/password
+combination. Instead of hardcoding it in the script you should request them
+from the keyring. In order to do this you need to request permissions to those
+credentials.
+
+Managing the keyring
+--------------------
+
+The keyring is a simple namespaced key-value store::
+
+    [sn0int][default] > keyring add aws:AKIAIOSFODNN7EXAMPLE
+    Secretkey: keep-this-secret
+    [sn0int][default] > keyring list
+    aws:AKIAIOSFODNN7EXAMPLE
+    [sn0int][default] >
+    [sn0int][default] > keyring list aws
+    aws:AKIAIOSFODNN7EXAMPLE
+    [sn0int][default] > keyring list instagram
+    [sn0int][default] >
+    [sn0int][default] > keyring get aws:AKIAIOSFODNN7EXAMPLE
+    Namespace:    "aws"
+    Access Key:   "AKIAIOSFODNN7EXAMPLE"
+    Secret:       "keep-this-secret"
+    [sn0int][default] >
+
+If the service uses a username-password combination, set the username as the
+access key and the password as the secret.
+
+If the service uses only a secret key for the api, set the secret key as the
+access key and leave the secret blank.
+
+A script doesn't automatically get access to requested keyring namespaces.
+Instead the user is asked to confirm those requests to limit abusive scripts.
+
+Using access keys in scripts
+----------------------------
+
+We can request all keys of a certain namespace in our script metadata. This is
+going to prompt the user to grant the script access. This can be done for
+multiple namespaces in the same script:
+
+.. code-block:: lua
+
+    -- Keyring-Access: aws
+    -- Keyring-Access: asdf
+
+If the user granted us access to those keys we can read them with ``keyring``:
+
+.. code-block:: lua
+
+    creds = keyring('aws')
+    debug(creds[1]['access_key'])
+    debug(creds[1]['secret_key'])
+
+This returns a list of all keys in that namespace. Any empty list is returned
+if the user doesn't have any keys in that namespace.
+
+Using access keys as source argument
+------------------------------------
+
+We can also use the access keys as source argument. This is useful if each
+account has access to different things and we want to read through all of them.
+
+Since access key permissions are granted per namespace we need to specify which
+credentials we want to use.
+
+.. code-block:: lua
+
+    -- Keyring-Access: aws
+    -- Source: keyring:aws

docs/man.rst

@@ -10,4 +10,5 @@ todo
     :glob:
 
     usage
+    config
     reference

docs/reference.rst

@@ -13,6 +13,28 @@ Clear the last recorded error from the internal state. See also last_err_.
         clear_err()
     end
 
+create_blob
+-----------
+
+Push a byte array into persistent blob storage. This allows passing those bytes
+to functions operating on blob storage. Returns a blob identifier that is
+deterministic based on the blob content. Blobs are immutable.
+
+.. code-block:: lua
+
+    blob = create_blob("some bytes")
+    debug(blob)
+
+datetime
+--------
+
+Return current time in UTC. This function is suitable to determine datetimes
+for ``DATETIME`` database fields.
+
+.. code-block:: lua
+
+    now = datetime()
+
 db_add
 ------
 
@@ -25,6 +47,23 @@ may fail or return ``nil``. See `db_add <database.html#db-add>`__ for details.
         value='example.com',
     })
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
 db_select
 ---------
 
@@ -55,14 +94,37 @@ Update an entity in the database. This function may fail. See `db_update
 dns
 ---
 
-Resolve a dns record. In the dns response contains an error, ``x['error']`` is
-set and ``x['success']`` is ``nil``. Otherwise, ``x['success']`` contains a
-list of records. This function may fail.
+Resolve a dns record. If the dns query was successful and the dns reply is
+``NoError`` then ``x['error']`` is ``nil``. The records of the reply are in
+``x['answers']``. This function may fail.
+
+This function accepts the following options:
+
+``record``
+  The ``query_type``, can be any of ``A``, ``AAAA``, ``MX``, ``AXFR``, etc.
+``nameserver``
+  The server that should be used for the lookup. Defaults to your system
+  resolver.
+``tcp``
+  If the lookup should use tcp, true/false.
+``timeout``
+  The time until the query times out in milliseconds.
 
 .. code-block:: lua
 
-    x = dns('example.com', 'A')
+    records = dns('example.com', {
+        record='A',
+    })
     if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+.. note::
+   DNS replies with an error code set are not causing a change to
+   ``last_err()``. You have to test for this explicitly.
+
+.. note::
+   This function is unavailable if a socks5 proxy is configured.
 
 error
 -----
@@ -147,19 +209,22 @@ Please note that you still need to specify an empty table ``{}`` even if no
 options are set. The following options are available:
 
 ``query``
-  A map of query parameters that should be set on the url
+  A map of query parameters that should be set on the url.
 ``headers``
-  A map of headers that should be set
+  A map of headers that should be set.
 ``basic_auth``
-  Configure the basic auth header with ``{"user, "password"}``
+  Configure the basic auth header with ``{"user, "password"}``.
 ``user_agent``
-  Overwrite the default user agent with a string
+  Overwrite the default user agent with a string.
 ``json``
-  The request body that should be json encoded
+  The request body that should be json encoded.
 ``form``
-  The request body that should be form encoded
+  The request body that should be form encoded.
 ``body``
-  The raw request body as string
+  The raw request body as string.
+``into_blob``
+  If true, the response body is stored in blob storage and a blob reference is
+  returned as ``blob`` instead of the full body.
 
 This function may fail.
 
@@ -187,6 +252,9 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``blob``
+  If ``into_blob`` was enabled for the request the body is downloaded into blob
+  storage with a reference to the body in this field.
 
 .. code-block:: lua
 
@@ -200,6 +268,41 @@ the following keys:
     if last_err() then return end
     if resp["status"] ~= 200 then return "invalid status code" end
 
+img_load
+--------
+
+Attempt to decode a blob as an image and return some basic metadata like the
+mime type, height and width.
+
+.. code-block:: lua
+
+    img = img_load(blob)
+    if last_err() then return end
+    debug(img)
+
+img_exif
+--------
+
+Extract exif metadata from an image.
+
+.. code-block:: lua
+
+    exif = img_exif(blob)
+    if last_err() then return end
+    debug(exif)
+
+img_nudity
+----------
+
+Classify an image for nudity. The score goes from 0 to 2. A score above 1 means
+nudity has been detected.
+
+.. code-block:: lua
+
+    nudity = img_nudity(blob)
+    if last_err() then return end
+    debug(nudity)
+
 info
 ----
 
@@ -243,10 +346,23 @@ Encode a datastructure into a string.
     })
     print(x)
 
+keyring
+-------
+
+Request all keys from a given namespace. See the `keyring <keyring.html>`__
+section for details.
+
+.. code-block:: lua
+
+    creds = keyring('aws')
+    print(creds[1]['accesskey'])
+    print(creds[1]['secretkey'])
+
 last_err
 --------
 
-Returns infos about the last error we've observed, if any. Returns ``nil`` otherwise.
+Returns infos about the last error we've observed, if any. Returns ``nil``
+otherwise.
 
 .. code-block:: lua
 
@@ -255,6 +371,15 @@ Returns infos about the last error we've observed, if any. Returns ``nil`` other
         return
     end
 
+md5
+---
+
+Hash a byte array with md5 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(md5("\x00\xff"))
+
 pgp_pubkey
 ----------
 
@@ -327,12 +452,12 @@ psl_domain_from_dns_name
 ------------------------
 
 Returns the parent domain according to the public suffix list. For
-``www.a.b.c.d.example.com`` this is going to be ``example.com``.
+``www.a.b.c.d.example.co.uk`` this is going to be ``example.co.uk``.
 
 .. code-block:: lua
 
-    domain = psl_domain_from_dns_name('www.a.b.c.d.example.com')
-    print(domain == 'example.com')
+    domain = psl_domain_from_dns_name('www.a.b.c.d.example.co.uk')
+    print(domain == 'example.co.uk')
 
 regex_find
 ----------
@@ -367,6 +492,33 @@ Same as regex_find_, but returns all matches.
     print(m[3][1] == 'ef')
     print(m[3][2] == 'f')
 
+sha1
+----
+
+Hash a byte array with sha1 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha1("\x00\xff"))
+
+sha2_256
+--------
+
+Hash a byte array with sha2_256 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha2_256("\x00\xff"))
+
+sha2_512
+--------
+
+Hash a byte array with sha2_512 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha2_512("\x00\xff"))
+
 sleep
 -----
 
@@ -377,6 +529,116 @@ only used for debugging.
 
     sleep(1)
 
+sock_connect
+------------
+
+Create a tcp connection.
+
+.. code-block:: lua
+
+    sock = sock_connect("127.0.0.1", 1337)
+
+sock_send
+---------
+
+Send data to the socket.
+
+.. code-block:: lua
+
+    sock_send(sock, "hello world")
+
+sock_recv
+---------
+
+Receive up to 4096 bytes from the socket.
+
+.. code-block:: lua
+
+    x = sock_recv(sock)
+
+sock_sendline
+-------------
+
+Send a string to the socket. A newline is automatically appended to the string.
+
+.. code-block:: lua
+
+    sock_sendline(sock, line)
+
+sock_recvline
+-------------
+
+Receive a line from the socket. The line includes the newline.
+
+.. code-block:: lua
+
+    x = sock_recvline(sock)
+
+sock_recvall
+------------
+
+Receive all data from the socket until EOF.
+
+.. code-block:: lua
+
+    x = sock_recvall(sock)
+
+sock_recvline_contains
+----------------------
+
+Receive lines from the server until a line contains the needle, then return
+this line.
+
+.. code-block:: lua
+
+    x = sock_recvline_contains(sock, needle)
+
+sock_recvline_regex
+-------------------
+
+Receive lines from the server until a line matches the regex, then return this
+line.
+
+.. code-block:: lua
+
+    x = sock_recvline_regex(sock, "^250 ")
+
+sock_recvn
+----------
+
+Receive exactly n bytes from the socket.
+
+.. code-block:: lua
+
+    x = sock_recvn(sock, 4)
+
+sock_recvuntil
+--------------
+
+Receive until the needle is found, then return all data including the needle.
+
+.. code-block:: lua
+
+    x = sock_recvuntil(sock, needle)
+
+sock_sendafter
+--------------
+
+Receive until the needle is found, then write data to the socket.
+
+.. code-block:: lua
+
+    sock_sendafter(sock, needle, data)
+
+sock_newline
+------------
+
+Overwrite the default ``\n`` newline.
+
+.. code-block:: lua
+
+    sock_newline(sock, "\r\n")
+
 status
 ------
 
@@ -386,6 +648,52 @@ Update the label of the progress indicator.
 
     status('ohai')
 
+stdin_readline
+--------------
+
+Read a line from stdin. The final newline is not removed.
+
+.. code-block:: lua
+
+    stdin_readline()
+
+.. note::
+   This only works with `sn0int run --stdin`.
+
+url_decode
+----------
+
+Parse a query string into a map. For raw percent decoding see url_unescape_.
+
+.. code-block:: lua
+
+    v = url_decode('a=b&c=d')
+    print(v['a'] == 'b')
+    print(v['c'] == 'd')
+
+url_encode
+----------
+
+Encode a map into a query string. For raw percent encoding see url_escape_.
+
+.. code-block:: lua
+
+    v = url_encode({
+        a='b',
+        c='d',
+    })
+    print(v == 'a=b&c=d')
+
+url_escape
+----------
+
+Apply url escaping to a string.
+
+.. code-block:: lua
+
+    v = url_escape('foo bar?')
+    print(v == 'foo%20bar%3F')
+
 url_join
 --------
 
@@ -415,11 +723,21 @@ Parse a url into its components. The following components are returned:
 
 .. code-block:: lua
 
-    url = url_parse("https://example.com")
+    url = url_parse('https://example.com')
     print(url['scheme'] == 'https')
     print(url['host'] == 'example.com')
     print(url['path'] == '/')
 
+url_unescape
+------------
+
+Remove url escaping of a string.
+
+.. code-block:: lua
+
+    v = url_unescape('foo%20bar%3F')
+    print(v == 'foo bar?')
+
 utf8_decode
 -----------
 

docs/sandbox.rst

@@ -0,0 +1,147 @@
+Sandbox
+=======
+
+Scripts are generally considered to be untrusted and executed exclusively in a
+child process. It's important to note that there's a basic sandbox that's
+active on every operating system, and there's a second line of defense on
+supported operating systems.
+
+The first line of defense is the restrictive stdlib. It's assumed that and
+attacker gains full control over the lua code and is able to call any function
+with arbitrary arguments. The stdlib only provides functions that are
+considered safe, so for example it's not possible to start a process or open a
+file.
+
+The second line of defense is supposed to make sure the system isn't
+compromised even if the first layer is fully broken and an attacker gains full
+control over the child process.
+
+Right now this is only supported on linux and openbsd.
+
+Linux
+-----
+
+On linux we use seccomp to filter all syscalls that we don't need. We also use
+chroot to disable filesystem access. It's recommended to install the sn0int
+binary with ``cap_sys_chroot`` to make sure unprivileged users can use chroot.
+The chroot location is hard coded and all capabilities are removed after the
+chroot is done or if no chroot is going to happen.
+
+OpenBSD
+-------
+
+On openbsd we're using ``pledge`` to restrict syscalls and ``unveil`` to
+restrict filesystem access.
+
+IPC Protocol
+------------
+
+The parent process and the child process communicate using an IPC protocol that
+is line-based json.
+
+For a simple hello world the parent process is only going to send a single line
+to the child process. This line contains:
+
+- The function argument
+- The dns config
+- Keys that the module has been given access to
+- The module metadata and code
+- Options, if any
+- A socks5 proxy, if any
+- The log level
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    -- nothing to do here\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+
+Saving this line in a file called ``start.json`` and sending it to a sandbox
+process should result in the following output::
+
+    $ sn0int sandbox foobar < start.json
+    {"Exit":"Ok"}
+    $
+
+This line tells us that the script terminated successfully.
+
+There are some functions that cause a notification to the parent process. We
+are going to add a call to the ``info()`` function to our module:
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    info('ohai')\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+
+This is going to print an additional event::
+
+    $ sn0int sandbox foobar < start2.json
+    {"Log":{"Info":"\"ohai\""}}
+    {"Exit":"Ok"}
+    $
+
+There are some functions that block the child process until the parent process
+sent a reply. These functions are mostly database related functions, since the
+child doesn't have direct database access. To demonstrate this, we're going to
+write two lines to our file this time, one is the init line and the second one
+is the reply for the database event:
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    x = db_add('domain', {value=\"example.com\"})\n    info(x)\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+    {"Ok":1337}
+
+Results in the following output::
+
+    $ target/release/sn0int sandbox foobar < start3.json
+    {"Database":{"Insert":{"Domain":{"value":"example.com"}}}}
+    {"Log":{"Info":"1337.0"}}
+    {"Exit":"Ok"}
+    $
+
+The first line is a database event and indicates that the child wants to insert
+data. After printing this line the child tries to read a line from stdin, this
+is why we needed to write two lines to our json file this time. In the second
+line the child learns if the insert was successful and which id was assigned to
+that entity.
+
+Limitations
+-----------
+
+There are some limitations that you should be aware:
+
+- Network access is available and network namespaces aren't isolated. This
+  means scripts have access to your local network, the internet and also your
+  localhost loopback interface.
+- If chroot is unavailable an attacker could connect to unix domain sockets.
+
+Diagnosing a sandbox failure
+----------------------------
+
+You might experience a sandbox failure, especially on architectures that are
+less popular. This usually looks like this::
+
+    [sn0int][example][kpcyrd/ctlogs] > run
+    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [+] Finished kpcyrd/ctlogs (1 errors)
+
+A module that never finishes could also mean an IO thread inside the worker got
+killed by the sandbox.
+
+You can try to diagnose this yourself with strace::
+
+    strace -f sn0int run -vv ctlogs 2>&1 | tee strace.log
+
+Open ``strace.log``, look out for syscalls that didn't return by searching for
+``= ?`` and ignore calls to exit and similar. You are looking for something
+like this::
+
+    seccomp(SECCOMP_SET_MODE_FILTER, 0, {len=48, filter=0xdd59094e490}) = 0
+    write(1, "[+] activated!\n", 15[+] activated!
+    )        = 15
+    getresuid( <unfinished ...>)            = ?
+    +++ killed by SIGSYS (core dumped) +++
+
+This would indicate a call to ``getresuid`` which was not allowed by the
+seccomp filter.
+
+If you don't want to diagnose this yourself open a new bug report with as much
+information as possible, specifically which distro, which release and which
+architecture you're using.

docs/scripting.rst

@@ -13,13 +13,29 @@ free to change that to something else::
 
     $ git init ~/repos/sn0int-modules
     $ cd ~/repos/sn0int-modules
-    $ ln -s "$PWD" ~/.local/share/sn0int/modules/$YOUR_GITHUB_NAME
+
+We need to add this folder to the sn0int config file so it's correctly detected
+when starting sn0int. Open the `config file <config.html>`_ in your prefered
+editor. Note that the file does not exist by default and the path is different
+depending on your operating system. On linux you would open the config file
+with::
+
+    $ vim ~/.config/sn0int.toml
+
+Add the follwing::
+
+    [namespaces]
+    your_github_name = "~/repos/sn0int-modules"
 
 Every module we're adding to ``~/repos/sn0int-modules`` is now going to be
 picked up by sn0int.
 
-Let's add our first module by opening ``~/repos/sn0int-modules/first.lua``.
-There's a bit of boilerplate that every module needs to load successfully:
+Make sure you're still in the right folder and add your first module::
+
+    sn0int new first.lua
+
+This is going to generate some boilerplate for you that every module needs to
+load successfully. Afterwards we can edit it like this:
 
 .. code-block:: lua
 
@@ -101,26 +117,31 @@ truth-y.
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
-        records = dns(subdomain, 'A')
+        records = dns(subdomain, {
+            record='A'
+        })
         if last_err() then return end
 
         print(records)
     end
 
 If you run your module again you're going to see some output, either
-``{"success": somedata}`` or ``{"error": "NX"}``. We decide if ``success`` is
-not ``nil``, we add the subdomain to our scope and set it to resolvable:
+``{"answers":[somedata],"error":null}`` or
+``{"answers":[],"error":"NXDomain"}``. We decide that we add the subdomain to
+our scope and set it to resolvable if ``error`` is ``nil``.
 
 .. code-block:: lua
 
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
-        records = dns(subdomain, 'A')
+        records = dns(subdomain, {
+            record='A'
+        })
         if last_err() then return end
 
-        if records['success'] ~= nil then
-            db_add('subdomain', arg, {
+        if records['error'] == nil then
+            db_add('subdomain', {
                 domain_id=arg['id'],
                 value=subdomain,
                 resolvable=true,
@@ -148,11 +169,13 @@ After putting everything together, our final module looks like this:
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
-        records = dns(subdomain, 'A')
+        records = dns(subdomain, {
+            record='A'
+        })
         if last_err() then return end
 
         if records['success'] ~= nil then
-            db_add('subdomain', arg, {
+            db_add('subdomain', {
                 domain_id=arg['id'],
                 value=subdomain,
                 resolvable=true,
@@ -181,3 +204,40 @@ your identity.
 Afterwards publish your module with::
 
     sn0int publish ./first.lua
+
+Reading data from stdin
+-----------------------
+
+Sometimes you need to read data that can't be easily accessed from within the
+sandbox, like output of other programms or file content. In that case you can
+write a module that reads from stdin:
+
+.. code-block:: lua
+
+    -- Description: Read from stdin
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+
+    function run()
+        while true do
+            x = stdin_readline()
+            if x == nil then
+                break
+            end
+            info(x)
+        end
+    end
+
+Write it to a file and run it like this::
+
+    % echo hello | sn0int run --stdin -vvf stdin.lua
+    [*] anonymous/stdin                                   : "hello\n"
+    [+] Finished anonymous/stdin
+    %
+
+This is going to read one line at a time and allows you to process it with
+regular expressions and add data to the database.
+
+.. note::
+   If you get an error like ``Failed to read stdin: "stdin is unavailable"``
+   make sure the ``--stdin`` flag is set.

docs/structs.rst

@@ -0,0 +1,254 @@
+Structs
+=======
+
+This section describes all supported structs in depth. Please refer to this
+section if in doubt about the correct usage of fields to ensure
+interoperability between modules.
+
+Domains
+-------
+
+Represents a registerable domain as defined by the `public suffix list
+<https://publicsuffix.org/>`_. If in doubt check `psl_domain_from_dns_name
+<reference.html#psl-domain-from-dns-name>`_.
+
+``value``
+  The domain name, like ``example.co.uk``.
+
+Subdomains
+----------
+
+A subdomain of a `domain <#domains>`_. The depth is arbitrary, so
+``foo.example.co.uk`` and ``foo.bar.example.co.uk`` are both valid subdomains
+of ``example.co.uk``.
+
+``value``
+  The subdomain, like ``foo.bar.example.co.uk``.
+``domain_id``
+  The numeric id of a domain struct.
+``resolvable``
+  Whether the subdomain can be resolved to a A/AAAA record. nil if unknown.
+
+IpAddrs
+-------
+
+An ip address. Note that most of these fields are geoip related and an
+approximation instead of an actual location.
+
+``value``
+    The ip address.
+``family``
+    The address family of the ip address, either ``4`` or ``6``.
+``continent``
+    The continent associated with this ip address.
+``continent_code``
+    The continent code of the ``continent`` field, eg ``NA``.
+``country``
+    The country associated with this ip address.
+``country_code``
+    The country code of the ``country`` field, eg ``US``.
+``city``
+    The city associated with this ip address.
+``latitude``
+    Latitude associated with this ip address.
+``longitude``
+    Longitude associated with this ip address.
+``asn``
+    The number of the autonomous system this ip belongs to.
+``as_org``
+    The organization of the autonomous system this ip belongs to.
+``description``
+    This field is sn0int internal if we have additional information about this
+    ip address, for example technical identifiers from aws.
+``reverse_dns``
+    The reverse dns name setup for this ip address.
+
+URLs
+----
+
+``subdomain_id``
+    The numeric id of a subdomain struct.
+``value``
+    The url, including a schema, hostname and path.
+``status``
+    The http status code, like ``200``.
+``body``
+    The raw response body. This can be any mime type.
+``online``
+    Whether or not the url gives a http response (even if it's an error).
+``title``
+    The parsed ``<title>`` of the page, if available.
+``redirect``
+    If the server replied with a redirect, this is the url it redirected to.
+
+Emails
+------
+
+``value``
+    The email address.
+``displayname``
+    The display name of a given email address: ``this is the name <foo@example.com>``.
+``valid``
+    Whether that email address is valid or has been disabled.
+
+Phonenumbers
+------------
+
+``value``
+    The phone number in E.164 format (+491234567)
+``name``
+    An alias we can assign to this phone number. This alias is sn0int internal.
+``valid``
+    Whether the number is assigned to a customer.
+``last_online``
+    The last time this number has been online.
+``country``
+    The country this number is associated with.
+``carrier``
+    The name of the carrier this numer is registered with.
+``line``
+    The type of the phone number, can be ``landline``, ``mobile`` or ``voip``.
+``is_ported``
+    Whether this number has been ported to a different carrier.
+``last_ported``
+    The last time this number has been ported.
+``caller_name``
+    The name of the owner of the phone number.
+``caller_type``
+    The type of caller, eg ``business`` or ``consumer``.
+
+Devices
+-------
+
+``value``
+    The devices mac address or another identifier if needed.
+``name``
+    An alias we can assign to this device. This alias is sn0int internal.
+``hostname``
+    The hostname configured on the device.
+``vendor``
+    The hardware vendor of the device. This is usually derived from the mac
+    address.
+``last_seen``
+    The last time we've observed the device somewhere.
+
+Networks
+--------
+
+A wired or wireless network at a specific location that a device could be
+connected to.
+
+``value``
+    The network name. This can be an ssid or any other identifier but should be
+    unique.
+``latitude``
+    Latitude of the networks location.
+``longitude``
+    Longitude of the networks location.
+
+Accounts
+--------
+
+A users account or profile on a webservice, like github or instagram.
+
+``service``
+    The identifier of the service/website. It's recommended to use the websites
+    domain for this as defined in `Domains`_.
+``username``
+    The users unique identifier, like the login name. If the login name is not
+    known or the system doesn't use login names, use the email address instead.
+``displayname``
+    The users display name. This name is often not unique and may contain the
+    users real name.
+``email``
+    The email address associated with the account.
+``url``
+    The url of the public profile if available.
+``last_seen``
+    The last time this account has been active/online.
+
+Breaches
+--------
+
+Either a breach of a specific website, a breach compilation or a breach
+notification service.
+
+``value``
+    The name of the breach, breach compilation or notification service.
+
+Images
+------
+
+``value``
+    The id that identifies the blob. This id is deterministic based on file
+    content.
+``filename``
+    This field is used if we have a well known filename for the content.
+``mime``
+    The image mimetype, like ``image/png`` or ``image/jpeg``.
+``width``
+    The width of the image.
+``height``
+    The height of the image.
+``created``
+    The date and time this image has been taken.
+``latitude``
+    Latitude this picture has been taken.
+``longitude``
+    Longitude this picture has been taken.
+``nudity``
+    A score that classifies nudity in this picture. The score goes from 0 to 2
+    and is commonly calculated with ``img_nudity``. A score above 1 means
+    nudity has been detected.
+``ahash``
+    The Mean (aHash) perceptual hash.
+``dhash``
+    The Gradient (dHash) perceptual hash.
+``phash``
+    The DCT (pHash) perceptual hash.
+
+Relations
+---------
+
+Relations are linking two structs together. The link may contain additional information.
+
+subdomain_ipaddr
+~~~~~~~~~~~~~~~~
+
+Links an ip address to a subdomain.
+
+``subdomain_id``
+    The numeric id of a subdomain struct.
+``ip_addr_id``
+    The numeric id of an ip addr struct.
+
+network_device
+~~~~~~~~~~~~~~
+
+Links a device to a network. This is commonly used with ``db_add_ttl`` so the
+link automatically expires. This is frequently used to monitor networks for
+known and unknown devices.
+
+``network_id``
+    The numeric id of a network struct.
+``device_id``
+    The numeric id of a device struct.
+``ipaddr``
+    The ip address assigned to the device.
+``last_seen``
+    The last time we've seen the device on that network.
+
+breach_email
+~~~~~~~~~~~~
+
+Links an email to a breach. If we know the password as well we can add it to
+the link. If we don't know the password we can leave it blank and fill it
+later. An email can be linked to a breach multiple times with different
+passwords. There is a special upserting logic in place to support this.
+
+``breach_id``
+    The numeric id of a breach struct.
+``email_id``
+    The numeric id of an email struct.
+``password``
+    The password for that email in the breach.

examples/asn.rs

@@ -1,32 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate maxminddb;
-
-use std::env;
-use sn0int::errors::*;
-use sn0int::geoip::{AsnDB, Maxmind};
-
-
-fn run() -> Result<()> {
-    let asndb = AsnDB::open_or_download()?;
-
-    for arg in env::args().skip(1) {
-        let ip = arg.parse()?;
-        let asn = asndb.lookup(ip)?;
-        println!("{:#?}", asn);
-    }
-
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}

examples/geoip.rs

@@ -1,31 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-
-use std::env;
-use sn0int::errors::*;
-use sn0int::geoip::{GeoIP, Maxmind};
-
-
-fn run() -> Result<()> {
-    let geoip = GeoIP::open_or_download()?;
-
-    for arg in env::args().skip(1) {
-        let ip = arg.parse()?;
-        let lookup = geoip.lookup(ip)?;
-        println!("{:#?}", lookup);
-    }
-
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}

examples/maxmind-dl.rs

@@ -1,50 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
-use sn0int::errors::*;
-use sn0int::geoip::{GeoIP, Maxmind};
-use sn0int::paths;
-use std::fs;
-use structopt::StructOpt;
-
-#[derive(Debug, StructOpt)]
-pub struct Args {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-fn run() -> Result<()> {
-    let args = Args::from_args();
-    debug!("{:?}", args);
-    let path = paths::cache_dir()?.join(&args.target);
-    if args.extract_only {
-        let body = fs::read(&args.url)?;
-        sn0int::archive::extract(&mut &body[..], &args.filter, path)?;
-    } else {
-        GeoIP::download(path, &args.filter, &args.url)?;
-    }
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}

examples/maxmind.rs

@@ -0,0 +1,104 @@
+extern crate sn0int;
+extern crate env_logger;
+extern crate chrootable_https;
+#[macro_use] extern crate log;
+
+// workaround for rustc 1.29.2 support
+#[cfg(not(target_os = "openbsd"))]
+extern crate structopt;
+#[cfg(target_os = "openbsd")]
+#[macro_use] extern crate structopt;
+
+use sn0int::errors::*;
+use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
+use sn0int::paths;
+use std::fs;
+use std::net::IpAddr;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="dl")]
+    Download(Download),
+    #[structopt(name="asn")]
+    Asn(AsnArgs),
+    #[structopt(name="geoip")]
+    GeoIP(GeoIPArgs),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Download {
+    url: String,
+    filter: String,
+    target: String,
+    #[structopt(short="e", long="extract-only")]
+    extract_only: bool,
+}
+
+impl Download {
+    fn run(&self) -> Result<()> {
+        let path = paths::cache_dir()?.join(&self.target);
+        if self.extract_only {
+            let body = fs::read(&self.url)?;
+            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
+        } else {
+            GeoIP::download(path, &self.filter, &self.url)?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AsnArgs {
+    ip: IpAddr,
+}
+
+impl AsnArgs {
+    fn run(&self) -> Result<()> {
+        let asndb = AsnDB::open_or_download()?;
+
+        let asn = asndb.lookup(self.ip)?;
+        println!("{:#?}", asn);
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct GeoIPArgs {
+    ip: IpAddr,
+}
+
+impl GeoIPArgs {
+    fn run(&self) -> Result<()> {
+        let geoip = GeoIP::open_or_download()?;
+
+        let lookup = geoip.lookup(self.ip)?;
+        println!("{:#?}", lookup);
+
+        Ok(())
+    }
+}
+
+
+fn run() -> Result<()> {
+    let args = Args::from_args();
+    debug!("{:?}", args);
+    match args {
+        Args::Download(args) => args.run(),
+        Args::Asn(args) => args.run(),
+        Args::GeoIP(args) => args.run(),
+    }
+}
+
+fn main() {
+    env_logger::init();
+
+    if let Err(err) = run() {
+        eprintln!("Error: {}", err);
+        for cause in err.iter_chain().skip(1) {
+            eprintln!("Because: {}", cause);
+        }
+        std::process::exit(1);
+    }
+}

examples/spinners.rs

@@ -1,20 +1,67 @@
-extern crate sn0int;
-
-use std::env;
 use std::thread;
 use std::time::Duration;
-use sn0int::term::{SPINNERS, Spinner};
+use sn0int::term::{SPINNERS, Spinner, StackedSpinners};
+use structopt::StructOpt;
+
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="single")]
+    Single(Single),
+    #[structopt(name="stacked")]
+    Stacked(Stacked),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Single {
+    idx: usize,
+    #[structopt(long="ticks", default_value="100")]
+    ticks: usize,
+}
+
+impl Single {
+    fn run(&self) {
+        let mut s = Spinner::new(SPINNERS[self.idx], "Demo".to_string());
+
+        for _ in 0..self.ticks {
+            thread::sleep(Duration::from_millis(100));
+            s.tick();
+        }
+
+        s.finish("Done".to_string());
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Stacked {
+}
+
+impl Stacked {
+    fn run(&self) {
+        let mut stack = StackedSpinners::new();
+        stack.add("1".into(), String::from("spinner1"));
+        stack.add("2".into(), String::from("spinner2"));
+        stack.add("3".into(), String::from("spinner3"));
+
+        for x in 1..=3 {
+            for _ in 0..50 {
+                thread::sleep(Duration::from_millis(100));
+                stack.tick();
+            }
+            // stack.log("ohai");
+            stack.remove(&x.to_string());
+        }
+
+        stack.clear();
+
+        // stack.finish("Done".to_string());
+    }
+}
 
 fn main() {
-    let idx = env::args().skip(1).next().expect("Expected argv[1]");
-    let idx = idx.parse::<usize>().expect("argv[1] is not a number");
-
-    let mut s = Spinner::new(SPINNERS[idx], "Demo".to_string());
-
-    for _ in 0..100 {
-        thread::sleep(Duration::from_millis(100));
-        s.tick();
+    let args = Args::from_args();
+    match args {
+        Args::Single(args) => args.run(),
+        Args::Stacked(args) => args.run(),
     }
-
-    s.finish("Done".to_string());
 }

examples/stacked-spinners.rs

@@ -1,25 +0,0 @@
-extern crate sn0int;
-
-use sn0int::term::StackedSpinners;
-use std::thread;
-use std::time::Duration;
-
-fn main() {
-    let mut stack = StackedSpinners::new();
-    stack.add("1".into(), String::from("spinner1"));
-    stack.add("2".into(), String::from("spinner2"));
-    stack.add("3".into(), String::from("spinner3"));
-
-    for x in 1..=3 {
-        for _ in 0..50 {
-            thread::sleep(Duration::from_millis(100));
-            stack.tick();
-        }
-        // stack.log("ohai");
-        stack.remove(&x.to_string());
-    }
-
-    stack.clear();
-
-    // stack.finish("Done".to_string());
-}

migrations/2018-10-26-001932_cascade-delete/up.sql

@@ -37,25 +37,6 @@ INSERT INTO subdomains (id, domain_id, value, unscoped, resolvable)
 
 DROP TABLE _subdomains_old;
 
--- subdomain_ipaddrs
-
-ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
-
-CREATE TABLE subdomain_ipaddrs (
-    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-    subdomain_id INTEGER NOT NULL,
-    ip_addr_id INTEGER NOT NULL,
-    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
-    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
-    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
-);
-
-INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
-    SELECT id, subdomain_id, ip_addr_id
-    FROM _subdomain_ipaddrs_old;
-
-DROP TABLE _subdomain_ipaddrs_old;
-
 -- urls
 
 ALTER TABLE urls RENAME TO _urls_old;
@@ -123,4 +104,23 @@ INSERT INTO ipaddrs (id, family, value, unscoped, continent, continent_code, cou
 
 DROP TABLE _ipaddrs_old;
 
+-- subdomain_ipaddrs
+
+ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
+
+CREATE TABLE subdomain_ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
+);
+
+INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
+    SELECT id, subdomain_id, ip_addr_id
+    FROM _subdomain_ipaddrs_old;
+
+DROP TABLE _subdomain_ipaddrs_old;
+
 PRAGMA foreign_keys=on;

migrations/2018-12-03-143558_url_path/down.sql

@@ -0,0 +1,25 @@
+PRAGMA foreign_keys=off;
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    title VARCHAR,
+    redirect VARCHAR,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online, title, redirect)
+  SELECT id, subdomain_id, value, status, body, unscoped, online, title, redirect
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-03-143558_url_path/up.sql

@@ -0,0 +1,26 @@
+PRAGMA foreign_keys=off;
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    path VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    title VARCHAR,
+    redirect VARCHAR,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, path, status, body, unscoped, online, title, redirect)
+  SELECT id, subdomain_id, value, '/', status, body, unscoped, online, title, redirect
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-14-094011_phonenumbers/down.sql

@@ -0,0 +1 @@
+DROP TABLE phonenumbers;

migrations/2018-12-14-094011_phonenumbers/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE phonenumbers (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    name VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    last_online DATETIME,
+    country VARCHAR,
+    carrier VARCHAR,
+    line VARCHAR,
+    is_ported BOOLEAN,
+    last_ported DATETIME,
+    caller_name VARCHAR,
+    caller_type VARCHAR,
+    CONSTRAINT phonenumber_unique UNIQUE (value)
+);

migrations/2018-12-23-230955_reverse-dns/down.sql

@@ -0,0 +1,27 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ipaddrs_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    asn INTEGER,
+    as_org VARCHAR,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO _ipaddrs_new (id, family, value, unscoped, continent, continent_code, city, latitude, longitude, asn, as_org)
+  SELECT id, family, value, unscoped, continent, continent_code, city, latitude, longitude, asn, as_org
+  FROM ipaddrs;
+
+DROP TABLE ipaddrs;
+ALTER TABLE _ipaddrs_new RENAME TO ipaddrs;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-23-230955_reverse-dns/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ipaddrs ADD COLUMN description VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN reverse_dns VARCHAR;

migrations/2018-12-24-141533_networks/down.sql

@@ -0,0 +1,3 @@
+DROP TABLE network_devices;
+DROP TABLE networks;
+DROP TABLE devices;

migrations/2018-12-24-141533_networks/up.sql

@@ -0,0 +1,30 @@
+CREATE TABLE networks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+CREATE TABLE devices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    name VARCHAR,
+    hostname VARCHAR,
+    vendor VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    last_seen DATETIME,
+    CONSTRAINT device_unique UNIQUE (value)
+);
+
+CREATE TABLE network_devices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    network_id INTEGER NOT NULL,
+    device_id INTEGER NOT NULL,
+    ipaddr VARCHAR,
+    last_seen DATETIME,
+    FOREIGN KEY(network_id) REFERENCES networks(id) ON DELETE CASCADE,
+    FOREIGN KEY(device_id) REFERENCES devices(id) ON DELETE CASCADE,
+    CONSTRAINT network_device_unique UNIQUE (network_id, device_id)
+);

migrations/2019-01-20-000235_ttl/down.sql

@@ -0,0 +1 @@
+DROP TABLE ttls;

migrations/2019-01-20-000235_ttl/up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE ttls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);

migrations/2019-02-03-123424_accounts/down.sql

@@ -0,0 +1 @@
+DROP TABLE accounts;

migrations/2019-02-03-123424_accounts/up.sql

@@ -0,0 +1,12 @@
+CREATE TABLE accounts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);

migrations/2019-02-11-011316_breaches/down.sql

@@ -0,0 +1,2 @@
+DROP TABLE breach_emails;
+DROP TABLE breaches;

migrations/2019-02-11-011316_breaches/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE breaches (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT breach_unique UNIQUE (value)
+);
+
+CREATE TABLE breach_emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    breach_id INTEGER NOT NULL,
+    email_id INTEGER NOT NULL,
+    password VARCHAR,
+    FOREIGN KEY(breach_id) REFERENCES breaches(id) ON DELETE CASCADE,
+    FOREIGN KEY(email_id) REFERENCES emails(id) ON DELETE CASCADE,
+    CONSTRAINT breach_emails_unique UNIQUE (breach_id, email_id, password)
+);

migrations/2019-03-08-060037_images/down.sql

@@ -0,0 +1 @@
+DROP TABLE images;

migrations/2019-03-08-060037_images/up.sql

@@ -0,0 +1,21 @@
+CREATE TABLE images (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+
+    filename VARCHAR,
+    mime VARCHAR,
+    width INT,
+    height INT,
+    created DATETIME,
+
+    latitude FLOAT,
+    longitude FLOAT,
+
+    nudity FLOAT,
+    ahash VARCHAR,
+    dhash VARCHAR,
+    phash VARCHAR,
+
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT image_unique UNIQUE (value)
+);

migrations/2019-03-24-195306_email-names/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _emails_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO _emails_new (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM emails;
+
+DROP TABLE emails;
+ALTER TABLE _emails_new RENAME TO emails;
+
+PRAGMA foreign_keys=on;

migrations/2019-03-24-195306_email-names/up.sql

@@ -0,0 +1,2 @@
+-- Your SQL goes here
+ALTER TABLE emails ADD COLUMN displayname VARCHAR;

modules/dev/arp-scan.lua

@@ -0,0 +1,45 @@
+-- Description: Parse arp-scan output
+-- Version: 0.3.0
+-- License: GPL-3.0
+
+-- sudo arp-scan -qglI wlp3s0
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        m = regex_find('(.+)\t(.+)', x)
+        if m ~= nil then
+            ipaddr = m[2]
+            mac = m[3]
+            now = datetime()
+
+            device_id = db_add('device', {
+                value=mac,
+                last_seen=now,
+            })
+            if last_err() then return end
+
+            db_add_ttl('network-device', {
+                network_id=network_id,
+                device_id=device_id,
+                ipaddr=ipaddr,
+                last_seen=now,
+            }, 300)
+            if last_err() then return end
+        end
+    end
+end

modules/dev/axfr.lua

@@ -0,0 +1,145 @@
+-- Description: Try a zone transfer for subdomains
+-- Version: 0.3.0
+-- Source: domains
+-- License: GPL-3.0
+
+function strip_root_dot(name)
+    local m = regex_find("(.+)\\.$", name)
+    if last_err() then return end
+
+    if m == nil then
+        return name
+    else
+        return m[2]
+    end
+end
+
+function add_pointer(name)
+    -- select psl+1
+    local domain = psl_domain_from_dns_name(name)
+    if last_err() then return end
+
+    -- add domain
+    local domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    -- add subdomain
+    local subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+    if last_err() then return end
+end
+
+function iter_axfr(zone, arg)
+    debug(arg)
+
+    local name = arg[1]
+    local r = arg[2]
+
+    -- select psl+1
+    local domain = psl_domain_from_dns_name(name)
+    if last_err() then return end
+
+    -- add domain
+    local domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    -- add subdomain
+    local subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+    if last_err() then return end
+
+    -- this is a A record
+    if r['A'] ~= nil then
+        -- add the name and ip
+        local ipaddr_id = db_add('ipaddr', {
+            family='4',
+            value=r['A'],
+        })
+        if last_err() then return end
+
+        db_add('subdomain-ipaddr', {
+            subdomain_id=subdomain_id,
+            ip_addr_id=ipaddr_id,
+        })
+        if last_err() then return end
+    end
+
+    if r['CNAME'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['CNAME'])
+        add_pointer(name)
+    end
+
+    if r['NS'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['NS'])
+        add_pointer(name)
+    end
+
+    if r['MX'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['MX'][2])
+        add_pointer(name:lower())
+    end
+end
+
+function iter_a(zone, arg)
+    if arg == nil then return end
+
+    debug('nameserver: ' .. arg)
+    local records = dns(zone, {
+        record='AXFR',
+        nameserver=arg .. ':53',
+        tcp=true,
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    for i=1, #records do
+        iter_axfr(zone, records[i])
+        if last_err() then return end
+    end
+end
+
+function iter_ns(zone, arg)
+    if arg == nil then return end
+
+    local records = dns(arg, {
+        record='A',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    for i=1, #records do
+        r = records[i][2]
+        iter_a(zone, r['A'])
+        if last_err() then return end
+    end
+end
+
+function run(arg)
+    local records = dns(arg['value'], {
+        record='NS',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    for i=1, #records do
+        local r = records[i][2]
+        iter_ns(arg['value'], r['NS'])
+        if last_err() then return end
+    end
+end

modules/dev/cname-harvest.lua

@@ -0,0 +1,48 @@
+-- Description: Query for CNAMES to find subdomains
+-- Version: 0.3.0
+-- Source: subdomains
+-- License: GPL-3.0
+
+function iter(r)
+    if r == nil then
+        return
+    end
+
+    m = regex_find("(.+)\\.$", r)
+    if last_err() then return end
+
+    if m == nil then
+        return
+    end
+    r = m[2]
+
+    domain = psl_domain_from_dns_name(r)
+    if last_err() then return end
+
+    domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+
+    if domain_id ~= nil then
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=r,
+        })
+        if last_err() then return end
+    end
+end
+
+function run(arg)
+    records = dns(arg['value'], 'A')
+    if last_err() then return end
+
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    for i=1, #records do
+        r = records[i][2]
+        iter(r['CNAME'])
+        if last_err() then return end
+    end
+end

modules/dev/ctlogs.lua

@@ -1,11 +1,59 @@
 -- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.1.0
+-- Version: 0.5.0
 -- Source: domains
 -- License: GPL-3.0
 
-function run(arg)
-    session = http_mksession()
+function each_name(name)
+    local domain_id, psl_domain
 
+    if seen[name] == 1 then
+        return
+    end
+    seen[name] = 1
+    debug(name)
+
+    if name:find('*.') == 1 then
+        -- ignore wildcard domains
+        return
+    end
+
+    -- the cert might be valid for subdomains that do not belong to the
+    -- domain we started with
+    psl_domain = psl_domain_from_dns_name(name)
+    domain_id = domains[psl_domain]
+    if domain_id == nil then
+        if any_domain then
+            -- unknown domains should be added to database
+            domain_id = db_add('domain', {
+                value=psl_domain,
+            })
+        else
+            -- only use domains that are already in scope
+            domain_id = db_select('domain', psl_domain)
+        end
+
+        -- if we didn't get a valid id, skip
+        if domain_id == nil then
+            return
+        end
+
+        domains[psl_domain] = domain_id
+    end
+
+    db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+end
+
+function run(arg)
+    full = getopt('full') ~= nil
+    any_domain = getopt('any-domain') ~= nil
+
+    domains = {}
+    domains[arg['value']] = arg['id']
+
+    session = http_mksession()
     req = http_request(session, 'GET', 'https://crt.sh/', {
         query={
             q='%.' .. arg['value'],
@@ -17,32 +65,37 @@ function run(arg)
     if last_err() then return end
     if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
 
-    certs = json_decode_stream(resp['text'])
+    certs = json_decode(resp['text'])
     if last_err() then return end
 
     seen = {}
 
-    i = 1
-    while i <= #certs do
+    for i=1, #certs do
         c = certs[i]
-        -- print(c)
+        debug(c)
 
-        name = c['name_value']
-
-        if name:find("*.") == 1 then
-            -- ignore wildcard domains
-            seen[name] = 1
-        end
-
-        if seen[name] == nil then
-            -- info(name)
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=name,
+        if full then
+            -- fetch certificate
+            id = c['min_cert_id']
+            req = http_request(session, 'GET', 'https://crt.sh/', {
+                query={
+                    d=id .. '', -- TODO: find nicer way for tostring
+                }
             })
-            seen[name] = 1
-        end
+            resp = http_send(req)
+            if last_err() then return end
+            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
 
-        i = i+1
+            -- iterate over all valid names
+            crt = x509_parse_pem(resp['text'])
+            if last_err() then return end
+            names = crt['valid_names']
+
+            for j=1, #names do
+                each_name(names[j])
+            end
+        else
+            each_name(c['name_value'])
+        end
     end
 end

modules/dev/ddwrt-dhcp.lua

@@ -0,0 +1,88 @@
+-- Description: Export dhcp leases from ddwrt webinterface
+-- Version: 0.2.0
+-- License: GPL-3.0
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    skip_redacted = not getopt('use-redacted')
+
+    router = getopt('router') -- http://192.0.2.1/
+    if not router then
+        return 'router option is missing (http://192.0.2.1/)'
+    end
+    username = getopt('user')
+    password = getopt('password')
+
+    options = {}
+    if username and password then
+        options['basic_auth'] = {username, password}
+    end
+
+    -- request status page
+    session = http_mksession()
+    url = url_join(router, '/Info.live.htm')
+    req = http_request(session, 'GET', url, options)
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then
+        return 'http error: ' .. resp['status']
+    end
+
+    txt = resp['text']
+    debug(txt)
+
+    -- extract leases from response
+    dhcp_section = regex_find('\\{dhcp_leases:: ([^\\}]+)\\}', txt)
+    if last_err() then return end
+    if not dhcp_section then
+        return 'Failed to get dhcp lease section'
+    end
+
+    leases = regex_find_all('\'([^\']+)\',\'([^\']+)\',\'([^\']+)\',\'[^\']+\',\'[^\']+\'', dhcp_section[2])
+    if last_err() then return end
+
+    now = datetime()
+
+    -- add devices to database
+    for i=1, #leases do
+        local hostname = leases[i][2]
+        local ipaddr = leases[i][3]
+        local macaddr = leases[i][4]
+
+        debug({
+            hostname=hostname,
+            ipaddr=ipaddr,
+            macaddr=macaddr,
+        })
+
+        if skip_redacted and macaddr:match('^xx:xx:') then
+            info('Skipping redacted macaddr')
+        else
+            local device = {
+                value=macaddr,
+                last_seen=now,
+            }
+            if hostname ~= '*' then
+                device['hostname'] = hostname
+            end
+
+            local device_id = db_add('device', device)
+
+            db_add_ttl('network-device', {
+                network_id=network_id,
+                device_id=device_id,
+                ipaddr=ipaddr,
+                last_seen=now,
+            }, 120)
+        end
+    end
+end

modules/dev/dns-ptr.lua

@@ -0,0 +1,30 @@
+-- Description: Run reverse dns lookups
+-- Version: 0.2.0
+-- Source: ipaddrs
+-- License: GPL-3.0
+
+function run(arg)
+    if arg['family'] == '4' then
+        m = regex_find('^(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)$', arg['value'])
+
+        q = m[5] .. '.' .. m[4] .. '.' .. m[3] .. '.' .. m[2] .. '.in-addr.arpa'
+        debug('Resolving: ' .. q)
+
+        records = dns(q, {
+            record='PTR',
+        })
+        if last_err() then return end
+        if records['error'] ~= nil then return end
+        records = records['answers']
+
+        for i=1, #records do
+            r = records[i][2]
+            if r['PTR'] then
+                db_update('ipaddr', arg, {
+                    reverse_dns=r['PTR'],
+                })
+                if last_err() then return end
+            end
+        end
+    end
+end

modules/dev/dns-resolve.lua

@@ -1,5 +1,5 @@
 -- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.1.0
+-- Version: 0.3.0
 -- Source: subdomains
 -- License: GPL-3.0
 
@@ -8,7 +8,7 @@ function run(arg)
     if last_err() then return end
 
     -- update subdomain
-    resolvable = records['success'] ~= nil
+    resolvable = records['error'] == nil
     if arg['resolvable'] ~= resolvable then
         -- TODO: pass arg to function as well
         db_update('subdomain', arg, {
@@ -20,15 +20,10 @@ function run(arg)
         return
     end
 
-    records = records['success']
+    records = records['answers']
 
-    -- there is a bug in struct -> lua that causes tables to be zero indexed
-    -- this checks if there's something at index 0 but uses index 1 if this is fixed
-    i = 0
-    if records[i] == nil then i = 1 end
-
-    while records[i] ~= nil do
-        r = records[i]
+    for i=1, #records do
+        r = records[i][2]
         if r['A'] ~= nil then
             ipaddr_id = db_add('ipaddr', {
                 family='4',
@@ -40,7 +35,7 @@ function run(arg)
                 subdomain_id=arg['id'],
                 ip_addr_id=ipaddr_id,
             })
+            if last_err() then return end
         end
-        i = i+1
     end
 end

modules/dev/exif.lua

@@ -0,0 +1,12 @@
+-- Description: Extract exif data from images
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    exif = img_exif(arg['value'])
+    if last_err() then return end
+    debug(exif)
+
+    db_update('image', arg, exif)
+end

modules/dev/git-webroot.lua

@@ -0,0 +1,28 @@
+-- Description: Search for git checkouts in webroot
+-- Version: 0.1.0
+-- Source: urls
+-- License: GPL-3.0
+
+function run(arg)
+    url = url_join(arg['value'], '.git/HEAD')
+
+    session = http_mksession()
+    req = http_request(session, 'GET', url, {})
+    reply = http_send(req)
+    if last_err() then return end
+
+    if reply['status'] ~= 200 then
+        return
+    end
+
+    if not regex_find('^ref: ', reply['text']) then
+        return
+    end
+
+    db_add('url', {
+        subdomain_id=arg['subdomain_id'],
+        value=url,
+        status=reply['status'],
+        body=reply['text'],
+    })
+end

modules/dev/github.lua

@@ -0,0 +1,107 @@
+-- Description: Collect data from github profiles
+-- Version: 0.2.0
+-- Source: accounts:github.com
+-- License: GPL-3.0
+
+function api_get(url)
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+    -- TODO: set_error(?)
+    if resp['status'] == 403 then return 'ratelimit exceeded' end
+    if resp['status'] ~= 200 then return 'invalid status code' end
+
+    local data = json_decode(resp['text'])
+    if last_err() then return end
+    return data
+end
+
+function import_gpg(url)
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+
+    local key = pgp_pubkey_armored(resp['text'])
+    if not key['uids'] then return end
+
+    for i=1, #key['uids'] do
+        local k = key['uids'][i]
+        debug(k)
+        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", k)
+        if m then
+            db_add('email', {
+                value=m[3],
+                displayname=m[2],
+            })
+        end
+    end
+end
+
+function scan4email(username)
+    local url = 'https://api.github.com/users/' .. username .. '/repos'
+    local repos = api_get(url)
+    if last_err() then return end
+
+    -- XXX: 'https://api.github.com/users/' .. username .. '/events/public?page=0&per_page=100' is faster but less accurate
+
+    for i=1, #repos do
+        local repo = repos[i]
+        debug(repo)
+        local commits = api_get(repo['url'] .. '/commits')
+        if last_err() then return end
+
+        for j=1, #commits do
+            local commit = commits[j]
+            debug(commit)
+
+            if commit['author'] and commit['author']['login'] == username then
+                local name = commit['commit']['author']['name']
+                local email = commit['commit']['author']['email']
+                db_add('email', {
+                    value=email,
+                    displayname=name,
+                })
+                return email
+            end
+
+            if commit['committer'] and commit['committer']['login'] == username then
+                local name = commit['commit']['committer']['name']
+                local email = commit['commit']['committer']['email']
+                db_add('email', {
+                    value=email,
+                    displayname=name,
+                })
+                return email
+            end
+        end
+    end
+end
+
+function run(arg)
+    session = http_mksession()
+    local url = 'https://api.github.com/users/' .. arg['username']
+
+    local data = api_get(url)
+    if last_err() then return end
+    debug(data)
+
+    -- company = data['company']
+    -- location = data['location']
+    -- homepage = data['blog']
+
+    url = 'https://github.com/' .. arg['username'] .. '.gpg'
+    import_gpg(url)
+    if last_err() then return end
+
+    local email = data['email']
+    if not email and not arg['email'] then
+        email = scan4email(arg['username'])
+        if last_err() then return end
+    end
+
+    db_update('account', arg, {
+        url=data['html_url'],
+        displayname=data['name'],
+        email=email,
+    })
+end

modules/dev/hackertarget-subdomains.lua

@@ -1,5 +1,5 @@
 -- Description: Query hackertarget for subdomains of a domain
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -18,13 +18,10 @@ function run(arg)
 
     m = regex_find_all("([^,]+),.+\\n?", resp['text'])
 
-    i = 1
-    while i <= #m do
+    for i=1, #m do
         db_add('subdomain', {
             domain_id=arg['id'],
             value=m[i][2]
         })
-
-        i = i+1
     end
 end

modules/dev/home-assistant-devices.lua

@@ -0,0 +1,56 @@
+-- Description: Query device location from home assistant
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Keyring-Access: home-assistant
+
+function run()
+    -- parsing options
+    instance = getopt('instance')
+    if not instance then
+        return 'instance option is missing'
+    end
+
+    host = url_parse(instance)
+    if last_err() then return end
+    host = host['host']
+
+    entity = getopt('entity')
+    if not entity then
+        return 'entity option is missing'
+    end
+
+    -- fetching credentials
+    creds = keyring('home-assistant:' .. host)
+    if creds[1] == nil then
+        profile = url_join(instance, 'profile')
+        return 'missing home-assistant:' .. host .. ' Long-Lived Access Token, open ' .. profile
+    end
+    token = creds[1]['secret_key']
+
+    headers = {}
+    headers['Authorization'] = 'Bearer ' .. token
+    headers['Content-Type'] = 'application/json'
+
+    -- requesting status
+    session = http_mksession()
+    url = url_join(instance, 'api/states/' .. entity)
+    req = http_request(session, 'GET', url, {
+        headers=headers
+    })
+    r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then
+        return 'http error: ' .. r['status']
+    end
+
+    m = json_decode(r['text'])
+    if last_err() then return end
+    debug(m)
+
+    info({
+        gps_accuracy=m['attributes']['gps_accuracy'],
+        longitude=m['attributes']['longitude'],
+        latitude=m['attributes']['latitude'],
+        last_updated=m['last_updated'],
+    })
+end

modules/dev/images.lua

@@ -0,0 +1,9 @@
+-- Description: Parse image metadata
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    local img = img_load(arg['value'])
+    db_update('image', arg, img)
+end

modules/dev/instagram.lua

@@ -0,0 +1,150 @@
+-- Description: Collect data from instagram profiles
+-- Version: 0.2.0
+-- Source: accounts:instagram.com
+-- License: GPL-3.0
+
+PAGE_SIZE = 50
+
+function get_shared_data(html)
+    local s = html_select_list(html, 'script')
+
+    for i=1, #s do
+        local m = regex_find('^window\\._sharedData = (.+);$', s[i]['text'])
+        if m then
+            return json_decode(m[2])
+        end
+    end
+end
+
+function sign_request(rhx, json_params)
+    local magic = rhx .. ':' .. json_params
+    local x_instagram_gis = hex(md5(magic))
+    return x_instagram_gis
+end
+
+function download_image(node)
+    local url = node['display_url']
+    debug(url)
+
+    local req = http_request(session, 'GET', url, {
+        into_blob=true,
+    })
+    local r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
+
+    db_add('image', {
+        value=r['blob'],
+    })
+end
+
+function pull_graphql(page)
+    local end_cursor = page['page_info']['end_cursor']
+
+    for i=1, #page['edges'] do
+        -- shortcode = page['edges'][i]['shortcode']
+        local node = page['edges'][i]['node']
+        node['thumbnail_resources'] = nil
+        node['media_preview'] = nil
+        -- debug(node)
+
+        -- if node['__typename'] == 'GraphImage'
+
+        -- node['dimensions']['height']
+        -- node['dimensions']['width']
+        -- ^ not sure how to get that picture
+
+        -- node['taken_at_timestamp']
+        -- location = node['location']
+
+        local err = download_image(node)
+        if last_err() then return end
+        if err ~= nil then return err end
+
+        todo_posts = todo_posts -1
+        debug('posts left: ' .. todo_posts .. '/' .. total_posts)
+    end
+
+    if page['page_info']['has_next_page'] then
+        debug('requesting next page=' .. end_cursor)
+
+        variables = json_encode({
+            id=user['id'],
+            first=PAGE_SIZE,
+            after=end_cursor
+        })
+
+        local headers = {}
+        headers['X-Instagram-GIS'] = sign_request(rhx_gis, variables)
+
+        local req = http_request(session, 'GET', 'https://www.instagram.com/graphql/query/', {
+            query={
+                query_hash='42323d64886122307be10013ad2dcc44',
+                variables=variables,
+            },
+            headers=headers,
+        })
+        r = http_send(req)
+        if last_err() then return end
+        if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
+
+        x = json_decode(r['text'])
+        if last_err() then return end
+        return pull_graphql(x['data']['user']['edge_owner_to_timeline_media'])
+    end
+end
+
+function run(arg)
+    session = http_mksession()
+    local url = 'https://www.instagram.com/' .. arg['username'] .. '/'
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'invalid status code' end
+    local html = resp['text']
+
+    local ld = html_select(html, 'script[type="application/ld+json"]')
+    if last_err() then return end
+
+    local ld = json_decode(ld['text'])
+    if last_err() then return end
+    --debug(ld)
+
+    if ld['email'] then
+        db_add('email', {
+            value=ld['email'],
+        })
+    end
+
+    -- homepage=ld['url']
+
+    db_update('account', arg, {
+        displayname=ld['name'],
+        email=ld['email'],
+        url=url,
+    })
+
+    -- download images
+    local sd = get_shared_data(html)
+    if last_err() then return end
+    -- debug(sd)
+
+    rhx_gis = sd['rhx_gis']
+    user = sd['entry_data']['ProfilePage'][1]['graphql']['user']
+
+    -- user['full_name']
+    -- user['id']
+    -- user['is_business_account']
+    -- user['is_private']
+    -- user['is_verified']
+    -- user['has_blocked_viewer']
+    -- user['connected_fb_page']
+    -- user['country_block']
+
+    local page = user['edge_owner_to_timeline_media']
+    total_posts = page['count']
+    todo_posts = total_posts
+
+    -- TODO: fast-update abort if image has been downloaded already
+    return pull_graphql(page)
+end

modules/dev/isc-dhcpd-leases.lua

@@ -0,0 +1,78 @@
+-- Description: Parse isc-dhcpd dhcpd.leases(5)
+-- Version: 0.2.0
+-- License: GPL-3.0
+
+-- cat /var/lib/dhcpd/dhcpd.leases
+
+function add(lease)
+    if not lease['active'] then return end
+
+    now = datetime()
+
+    device_id = db_add('device', {
+        value=lease['mac'],
+        hostname=lease['hostname'],
+        last_seen=now,
+    })
+    if last_err() then return end
+
+    db_add_ttl('network-device', {
+        network_id=network_id,
+        device_id=device_id,
+        ipaddr=lease['ipaddr'],
+        last_seen=now,
+    }, 180)
+    if last_err() then return end
+end
+
+function each_line(x)
+    debug(x)
+    m = regex_find('^lease (\\S+) \\{\n$', x)
+    if m then
+        lease = {}
+        debug('ipaddr=' .. m[2])
+        lease['ipaddr'] = m[2]
+    end
+    m = regex_find('^\\s*hardware ethernet (\\S+);\n$', x)
+    if m then
+        debug('mac=' .. m[2])
+        lease['mac'] = m[2]
+    end
+    m = regex_find('^\\s*client-hostname \"(.+)\";\n$', x)
+    if m then
+        debug('hostname=' .. m[2])
+        lease['hostname'] = m[2]
+    end
+    m = regex_find('^\\s*binding state active;\n$', x)
+    if m then
+        debug('active=true')
+        lease['active'] = true
+    end
+    m = regex_find('^\\}\n$', x)
+    if m then
+        add(lease)
+    end
+end
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        if not regex_find('^\\s*(#.*|\\s*)\n$', x) then
+            each_line(x)
+        end
+    end
+end

modules/dev/iw-station-dump.lua

@@ -0,0 +1,77 @@
+-- Description: Parse iw station dump
+-- Version: 0.2.0
+-- License: GPL-3.0
+
+-- iw dev wlan0 station dump
+
+function add(client)
+    if
+        client['authenticated'] == 'yes' and
+        client['authorized'] == 'yes' and
+        client['mac']
+    then
+        debug(client)
+
+        now = datetime()
+
+        device_id = db_add('device', {
+            value=client['mac'],
+            last_seen=now,
+        })
+        if last_err() then return end
+
+        db_add_ttl('network-device', {
+            network_id=network_id,
+            device_id=device_id,
+            last_seen=now,
+        }, 180)
+        if last_err() then return end
+    end
+
+    client = nil
+end
+
+function each_line(x)
+    debug(x)
+    m = regex_find('^Station (\\S+)', x)
+    if m then
+        if client then
+            add(client)
+        end
+        client = {}
+        client['mac'] = m[2]
+        debug('mac=' .. m[2])
+    end
+
+    m = regex_find('^\\s+([^:]+):\\s*(.+)\n$', x)
+    if m and client then
+        client[m[2]] = m[3]
+        debug(m[2] .. '=' .. m[3])
+    end
+end
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    client = nil
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        each_line(x)
+    end
+
+    if client then
+        add(client)
+    end
+end

modules/dev/keybase-domains.lua

@@ -0,0 +1,28 @@
+-- Description: Find keybase proofs for domains
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
+        query={
+            domain=arg['value'],
+        }
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    x = json_decode(resp['text'])
+    if last_err() then return end
+    debug(x)
+
+    if x['them'][1] == nil then return end
+    them = x['them'][1]
+
+    db_add('account', {
+        service='keybase.io',
+        username=them['basics']['username'],
+    })
+end

modules/dev/keybase-profiles.lua

@@ -0,0 +1,44 @@
+-- Description: Find keybase proofs for online accounts
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: accounts
+
+function run(arg)
+    service = arg['service']
+    if service == 'twitter.com' then
+        service = 'twitter'
+    elseif service == 'github.com' then
+        service = 'github'
+    elseif service == 'reddit.com' then
+        service = 'reddit'
+    elseif service == 'news.ycombinator.com' then
+        service = 'hackernews'
+    elseif service == 'facebook.com' then
+        service = 'facebook'
+    else
+        return
+    end
+
+    query = {}
+    query[service] = arg['username']
+
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
+        query=query,
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    x = json_decode(resp['text'])
+    if last_err() then return end
+    debug(x)
+
+    if x['them'][1] == nil then return end
+    them = x['them'][1]
+
+    db_add('account', {
+        service='keybase.io',
+        username=them['basics']['username'],
+    })
+end

modules/dev/keybase.lua

@@ -0,0 +1,85 @@
+-- Description: Collect accounts and emails from keybase accounts
+-- Version: 0.2.0
+-- License: GPL-3.0
+-- Source: accounts:keybase.io
+
+function extract_mails(pubkey)
+    for j=1, #pubkey['uids'] do
+        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
+        if m then
+            db_add('email', {
+                value=m[3],
+                displayname=m[2],
+            })
+        end
+    end
+end
+
+function add_domain(dns)
+    local domain = psl_domain_from_dns_name(dns)
+    if last_err() then return end
+
+    local domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    if domain ~= dns then
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=dns,
+        })
+    end
+end
+
+function run(arg)
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
+        query={
+            usernames=arg['username'],
+        }
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    x = json_decode(resp['text'])
+    if last_err() then return end
+    debug(x)
+
+    if x['them'][1] == nil then return end
+    them = x['them'][1]
+
+    -- update keybase profile
+    db_update('account', arg, {
+        displayname=them['profile']['full_name'],
+        url='https://keybase.io/'..arg['username'],
+    })
+
+    -- collect emails
+    pubkey = pgp_pubkey_armored(them['public_keys']['primary']['bundle'])
+    debug(pubkey)
+    extract_mails(pubkey)
+
+    -- collect profiles
+    profiles = them['proofs_summary']['all']
+
+    for i=1, #profiles do
+        profile = profiles[i]
+        debug(profile)
+
+        if
+            profile['proof_type'] == 'generic_web_site' or
+            profile['proof_type'] == 'dns'
+        then
+            add_domain(profile['nametag'])
+        else
+            db_add('account', {
+                service=profile['proof_type'],
+                username=profile['nametag'],
+                url=profile['service_url'],
+            })
+        end
+    end
+end

modules/dev/namechk.lua

@@ -0,0 +1,85 @@
+-- Description: Find accounts by username with namechk.com
+-- Version: 0.2.0
+-- Source: accounts
+-- License: GPL-3.0
+
+function get_services(html)
+    local divs = html_select_list(html, '.service')
+    if last_err() then return end
+
+    local services = {}
+
+    for i=1, #divs do
+        services[i] = divs[i]['attrs']['data-name']
+    end
+
+    return services
+end
+
+function run(arg)
+    -- setup session
+    local session = http_mksession()
+    local req = http_request(session, 'GET', 'https://namechk.com/', {})
+    local resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    local token = html_select(resp['text'], 'input[name="authenticity_token"]')
+    local auth_token = token['attrs']['value']
+
+        local headers = {}
+        headers['X-CSRF-Token'] = authenticity_token
+
+    local services = get_services(resp['text'])
+    debug({
+        auth_token=auth_token,
+        services=services,
+    })
+
+    -- trigger the scan
+    local req = http_request(session, 'POST', 'https://namechk.com/', {
+            headers=headers,
+        form={
+            authenticity_token=auth_token,
+            q=arg['username'],
+        }
+    })
+    local resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+    debug(resp)
+
+    local scan = json_decode(resp['text'])
+    if last_err() then return end
+    local scan_token = scan['valid']
+
+    -- get results
+    for i=1, #services do
+        debug(services[i])
+
+        local req = http_request(session, 'POST', 'https://namechk.com/services/check', {
+            headers=headers,
+            form={
+                token=scan_token,
+                fat=auth_token,
+                service=services[i],
+            }
+        })
+        local resp = http_send(req)
+        if last_err() then return end
+
+        if resp['status'] == 200 then
+            local acc = json_decode(resp['text'])
+            if last_err() then return end
+            debug(acc)
+
+            if acc ~= nil and not acc['available'] and acc['status'] == 'unavailable' then
+                db_add('account', {
+                    service=services[i],
+                    username=arg['username'],
+                    url=acc['callback_url'],
+                })
+            end
+        end
+    end
+end

modules/dev/nudity.lua

@@ -0,0 +1,12 @@
+-- Description: Scan collected images for nudity
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    local nudity = img_nudity(arg['value'])
+    debug(nudity)
+    db_update('image', arg, {
+        nudity=nudity['score'],
+    })
+end

modules/dev/otx-subdomains.lua

@@ -1,5 +1,5 @@
 -- Description: Query alienvault otx passive dns for subdomains of a domain
--- Version: 0.1.0
+-- Version: 0.3.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -18,15 +18,12 @@ function run(arg)
     if last_err() then return end
     o = o['passive_dns']
 
-    i = 0
-    while o[i] do
+    for i=1, #o do
         x = o[i]
 
         db_add('subdomain', {
             domain_id=arg['id'],
             value=x['hostname'],
         })
-
-        i = i+1
     end
 end

modules/dev/passive-arp.lua

@@ -0,0 +1,79 @@
+-- Description: Passive arp-scanner with sniffglue
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- sudo sniffglue -jv enp0s25
+
+function each_frame(frame)
+    if not frame['Ether'] then return end
+
+    local arp = frame['Ether'][2]['Arp']
+    if not arp then return end
+
+    if arp['Request'] then
+        arp = arp['Request']
+    elseif arp['Reply'] then
+        arp = arp['Reply']
+    else
+        -- unknown, abort
+        return
+    end
+
+    debug(arp)
+
+    -- TODO: this might change to a string in the future
+    local mac = mac(arp['src_mac'])
+    local ipaddr = arp['src_addr']
+    debug({src_mac=mac, src_addr=ipaddr})
+
+    local now = datetime()
+
+    local device_id = db_add('device', {
+        value=mac,
+        last_seen=now,
+    })
+    if last_err() then return end
+
+    db_add_ttl('network-device', {
+        network_id=network_id,
+        device_id=device_id,
+        ipaddr=ipaddr,
+        last_seen=now,
+    }, 120)
+    if last_err() then return end
+end
+
+function mac(m)
+    return
+        hex({m[1]}) .. ':' ..
+        hex({m[2]}) .. ':' ..
+        hex({m[3]}) .. ':' ..
+        hex({m[4]}) .. ':' ..
+        hex({m[5]}) .. ':' ..
+        hex({m[6]})
+end
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    while true do
+        local x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        local frame = json_decode(x)
+        if last_err() then return end
+
+        each_frame(frame)
+        if last_err() then return end
+    end
+end

modules/dev/passive-spider.lua

@@ -1,5 +1,5 @@
 -- Description: Scrape known http responses for urls
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: urls
 -- License: GPL-3.0
 
@@ -52,12 +52,9 @@ function run(arg)
     end
 
     -- process html links
-    i = 1
-    while i <= #links do
+    for i=1, #links do
         href = links[i]['attrs']['href']
 
         entry(arg['value'], href)
-
-        i = i+1
     end
 end

modules/dev/pgp-keyserver.lua

@@ -1,5 +1,5 @@
 -- Description: Query pgp keyserver for email addresses
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -20,8 +20,7 @@ function run(arg)
     if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
 
     links = html_select_list(resp['text'], 'a')
-    i = 1
-    while i <= #links do
+    for i=1, #links do
         href = links[i]['attrs']['href']
 
         if href:find('/pks/lookup%?op=get&search=') == 1 then
@@ -40,19 +39,16 @@ function run(arg)
 
             -- TODO: ensure at least one email matches our target domain
             if pubkey['uids'] then
-                j = 1
-                while j <= #pubkey['uids'] do
-                    m = regex_find("<([^< ]+@[^< ]+)>$", pubkey['uids'][j])
+                for j=1, #pubkey['uids'] do
+                    local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
                     if m then
                         db_add('email', {
-                            value=m[2],
+                            value=m[3],
+                            displayname=m[2],
                         })
                     end
-                    j = j+1
                 end
             end
         end
-
-        i = i+1
     end
 end

modules/dev/phpmyadmin.lua

@@ -0,0 +1,103 @@
+-- Description: Search for phpmyadmin
+-- Version: 0.2.0
+-- Source: urls
+-- License: GPL-3.0
+
+function run(arg)
+    paths = {
+        "phpmyadmin/index.php",
+        "phpMyAdmin/index.php",
+        "pmd/index.php",
+        "pma/index.php",
+        "PMA/index.php",
+        "PMA2/index.php",
+        "pmamy/index.php",
+        "pmamy2/index.php",
+        "mysql/index.php",
+        "admin/index.php",
+        "db/index.php",
+        "dbadmin/index.php",
+        "web/phpMyAdmin/index.php",
+        "admin/pma/index.php",
+        "admin/PMA/index.php",
+        "admin/mysql/index.php",
+        "admin/mysql2/index.php",
+        "admin/phpmyadmin/index.php",
+        "admin/phpMyAdmin/index.php",
+        "admin/phpmyadmin2/index.php",
+        "mysqladmin/index.php",
+        "mysql-admin/index.php",
+        "mysql_admin/index.php",
+        "phpadmin/index.php",
+        "phpAdmin/index.php",
+        "phpmyadmin0/index.php",
+        "phpmyadmin1/index.php",
+        "phpmyadmin2/index.php",
+        "phpMyAdmin-4.4.0/index.php",
+        "myadmin/index.php",
+        "myadmin2/index.php",
+        "xampp/phpmyadmin/index.php",
+        "phpMyadmin_bak/index.php",
+        "www/phpMyAdmin/index.php",
+        "tools/phpMyAdmin/index.php",
+        "phpmyadmin-old/index.php",
+        "phpMyAdminold/index.php",
+        "phpMyAdmin.old/index.php",
+        "pma-old/index.php",
+        "claroline/phpMyAdmin/index.php",
+        "typo3/phpmyadmin/index.php",
+        "phpma/index.php",
+        "phpmyadmin/phpmyadmin/index.php",
+        "phpMyAdmin/phpMyAdmin/index.php",
+        "phpMyAbmin/index.php",
+        "phpMyAdmin__/index.php",
+        "phpMyAdmin+++---/index.php",
+        "v/index.php",
+        "phpmyadm1n/index.php",
+        "phpMyAdm1n/index.php",
+        "shaAdmin/index.php",
+        "phpMyadmi/index.php",
+        "phpMyAdmion/index.php",
+        "MyAdmin/index.php",
+        "phpMyAdmin1/index.php",
+        "phpMyAdmin123/index.php",
+        "pwd/index.php",
+        "phpMyAdmina/index.php",
+        "program/index.php",
+        "shopdb/index.php",
+        "phppma/index.php",
+        "phpmy/index.php",
+        "mysql/admin/index.php",
+        "mysql/dbadmin/index.php",
+        "mysql/sqlmanager/index.php",
+        "mysql/mysqlmanager/index.php",
+        "wp-content/plugins/portable-phpmyadmin/wp-pma-mod/index.php",
+    }
+
+    session = http_mksession()
+
+    for i=1, #paths do
+        p = paths[i]
+        url = url_join(arg['value'], p)
+        debug(url)
+
+        req = http_request(session, 'GET', url, {
+            timeout=5000
+        })
+        reply = http_send(req)
+        debug(reply)
+
+        if last_err() then
+            clear_err()
+        else
+            if reply['status'] == 200 then
+                db_add('url', {
+                    subdomain_id=arg['subdomain_id'],
+                    value=url,
+                    status=reply['status'],
+                    body=reply['text'],
+                })
+            end
+        end
+    end
+end

modules/dev/smtp-check.lua

@@ -0,0 +1,88 @@
+-- Description: Verify email address by asking the smtp server
+-- Version: 0.2.0
+-- Source: emails
+-- License: GPL-3.0
+
+function find_mx(domain)
+    local records, i, r
+
+    records = dns(domain, {
+        record='MX',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+    -- debug(records)
+
+    for i=1, #records do
+        r = records[i][2]['MX']
+        if r then
+            debug('mx: ' .. r[2])
+            return r[2]
+        end
+    end
+end
+
+function run(arg)
+    -- extract domain
+    domain = arg['value']:match('@(.*)')
+    if doman ~= nil then
+        -- malformed domain
+        return
+    end
+
+    -- mx lookup
+    mx = find_mx(domain)
+    if last_err() then return end
+    if not mx then return end
+
+    -- create connection
+    c = sock_connect(mx, 25, {})
+    if last_err() then return end
+
+    l = sock_recvline(c)
+    if last_err() then return end
+    debug(l)
+
+    -- send hello
+    sock_sendline(c, 'ehlo localhost')
+    if last_err() then return end
+
+    l = sock_recvline_regex(c, '^250 ')
+    if last_err() then return end
+    debug(l)
+
+    -- send email
+    sock_sendline(c, 'mail from:<root@localhost>')
+    if last_err() then return end
+
+    l = sock_recvline(c)
+    if last_err() then return end
+    debug(l)
+
+    -- send rcpt
+    sock_sendline(c, 'rcpt to:<' .. arg['value'] .. '>')
+    if last_err() then return end
+
+    l = sock_recvline(c)
+    if last_err() then return end
+    debug(l)
+
+    -- check status
+    verified = nil
+    if l:match('^2') then
+        debug('email is valid')
+        verified = true
+    elseif l:match('^5') then
+        debug('email is invalid')
+        verified = false
+    elseif l:match('^4') then
+        debug('unknown status, temporary delivery failure')
+    end
+
+    if verified ~= nil then
+        db_update('email', arg, {
+            valid=verified,
+        })
+    end
+end

modules/dev/threatminer-ipaddr.lua

@@ -1,5 +1,5 @@
 -- Description: Query ThreatMiner passive dns for subdomains of an ip address
--- Version: 0.1.0
+-- Version: 0.3.0
 -- Source: ipaddrs
 -- License: GPL-3.0
 
@@ -23,8 +23,7 @@ function run(arg)
     if last_err() then return end
     o = o['results']
 
-    i = 0
-    while o[i] do
+    for i=1, #o do
         x = o[i]
 
         domain = psl_domain_from_dns_name(x['domain'])
@@ -46,7 +45,5 @@ function run(arg)
                 ip_addr_id=arg['id'],
             })
         end
-
-        i = i+1
     end
 end

modules/dev/threatminer-subdomains.lua

@@ -1,5 +1,5 @@
 -- Description: Query ThreatMiner passive dns for subdomains of a domain
--- Version: 0.1.0
+-- Version: 0.3.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -21,15 +21,12 @@ function run(arg)
     if last_err() then return end
     o = o['results']
 
-    i = 0
-    while o[i] do
+    for i=1, #o do
         x = o[i]
 
         db_add('subdomain', {
             domain_id=arg['id'],
             value=x,
         })
-
-        i = i+1
     end
 end

modules/dev/thunderbird-autoconfig.lua

@@ -0,0 +1,45 @@
+-- Description: Query thunderbird autoconfig db for subdomains
+-- Version: 0.2.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run(arg)
+    session = http_mksession()
+
+    -- check if an autoconfig exists without disclosing our target yet
+    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/', {})
+    resp = http_send(req)
+    if last_err() then return end
+
+    if resp['status'] ~= 200 then
+        return 'index request failed'
+    end
+
+    if resp['text']:find(arg['value'], 1, true) == nil then
+        debug('no autoconfig available')
+        return
+    end
+
+    -- request config
+    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/' .. arg['value'], {})
+    resp = http_send(req)
+    if last_err() then return end
+
+    m = regex_find_all('<hostname>([^<]+)</hostname>', resp['text'])
+
+    for i=1, #m do
+        subdomain = m[i][2]
+
+        domain = psl_domain_from_dns_name(subdomain)
+        if last_err() then return end
+
+        domain_id = db_select('domain', domain)
+        if last_err() then return end
+
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=subdomain,
+        })
+        if last_err() then return end
+    end
+end

modules/dev/twilio-lookup.lua

@@ -0,0 +1,48 @@
+-- Description: Retrieve additional information about a phone number
+-- Version: 0.1.0
+-- Source: phonenumbers
+-- Keyring-Access: twilio
+-- License: GPL-3.0
+
+function run(arg)
+    number = url_escape(arg['value'])
+    --url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number
+    url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number .. '?Type=carrier&Type=caller-name'
+
+    --debug(url)
+
+    key = keyring('twilio')[1]
+    if not key then
+        return 'Missing required twilio access key'
+    end
+
+    session = http_mksession()
+    req = http_request(session, 'GET', url, {
+        basic_auth={key['access_key'], key['secret_key']},
+    })
+    reply = http_send(req)
+    if last_err() then return end
+
+    if reply['status'] ~= 200 then
+        return 'api returned error'
+    end
+
+    v = json_decode(reply['text'])
+    if last_err() then return end
+    debug(v)
+
+    update = {}
+    update['country'] = v['country_code']
+
+    if v['carrier'] then
+        update['carrier'] = v['carrier']['name']
+        update['line'] = v['carrier']['type']
+    end
+
+    if v['caller_name'] then
+        update['caller_name'] = v['caller_name']['caller_name']
+        update['caller_type'] = v['caller_name']['caller_type']
+    end
+
+    db_update('phonenumber', arg, update)
+end

modules/dev/url-scan.lua

@@ -1,5 +1,5 @@
 -- Description: Scan subdomains for websites
--- Version: 0.1.0
+-- Version: 0.3.0
 -- Source: subdomains
 -- License: GPL-3.0
 
@@ -19,7 +19,6 @@ function request(subdomain_id, url)
         value=url,
         status=reply['status'],
         body=reply['text'],
-        redirect=reply['headers']['location'],
     }
 
     redirect = reply['headers']['location']
@@ -29,9 +28,9 @@ function request(subdomain_id, url)
 
     db_add('url', obj)
 
-    -- info(json_encode(reply['status']))
-    -- info(json_encode(reply['headers']['location']))
-    -- info(json_encode(reply['text']))
+    -- debug(reply['status'])
+    -- debug(reply['headers']['location'])
+    -- debug(reply['text'])
 end
 
 function run(arg)

modules/dev/waybackurls.lua

@@ -1,5 +1,5 @@
 -- Description: Discover subdomains from wayback machine
--- Version: 0.1.0
+-- Version: 0.4.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -18,21 +18,20 @@ function run(arg)
     if last_err() then return end
 
     -- no known urls
-    if o[0] == nil then
+    if o[1] == nil then
         return
     end
 
     -- ensure the api response is still what we expect
-    if o[0][2] == nil then
+    if o[1][3] == nil then
         return 'api returned unexpected json format'
     end
 
     seen = {}
 
-    i = 1
-    while o[i] do
-        url = o[i][2]
-
+    for i=2, #o do
+        url = o[i][3]
+        debug(url)
         parts = url_parse(url)
 
         if last_err() then
@@ -40,6 +39,7 @@ function run(arg)
             error("Failed to parse url: " .. json_encode(url))
         else
             subdomain = parts['host']
+            subdomain, _ = subdomain:gsub('%.$', '')
 
             if seen[subdomain] == nil then
                 db_add('subdomain', {
@@ -50,7 +50,5 @@ function run(arg)
                 seen[subdomain] = 1
             end
         end
-
-        i = i+1
     end
 end

modules/dev/well-known-uris.lua

@@ -0,0 +1,61 @@
+-- Description: Scan for known /.well-known/ locations
+-- Version: 0.2.0
+-- Source: urls
+-- License: GPL-3.0
+
+function run(arg)
+    -- https://www.iana.org/assignments/well-known-uris/well-known-uris.xhtml
+    -- https://en.wikipedia.org/wiki/List_of_/.well-known/_services_offered_by_webservers
+
+    -- TODO: check if every location causes a 200/redirect
+
+    locations = {
+        {path='security.txt'}, -- expect 200
+        {path='dnt-policy.txt'}, -- expect 200
+        {path='caldav', redirect=true}, -- expect redirect
+        {path='autoconfig/mail/config-v1.1.xml'}, -- expect 200
+        {path='assetlinks.json'}, -- expect 200
+        {path='apple-app-site-association'},  -- expect 200
+        {path='keybase.txt'}, -- expect 200
+        {path='apple-developer-merchantid-domain-association'}, -- expect 200
+        {path='openpgpkey'}, -- expect 200
+        {path='change-password', redirect=true}, -- expect redirect
+    }
+
+    session = http_mksession()
+
+    for i=1, #locations do
+        path = locations[i]['path']
+        expect_redirect = locations[i]['redirect']
+
+        url = url_join(arg['value'], '/.well-known/' .. path)
+        debug(url)
+
+        req = http_request(session, 'GET', url, {
+            timeout=5000,
+        })
+        reply = http_send(req)
+        debug(reply)
+
+        if last_err() then
+            clear_err()
+        else
+            status = reply['status']
+            if (status == 200 and not expect_redirect) or (expect_redirect and status >= 300 and status < 400) then
+                obj = {
+                    subdomain_id=arg['subdomain_id'],
+                    value=url,
+                    status=reply['status'],
+                    body=reply['text'],
+                }
+
+                redirect = reply['headers']['location']
+                if redirect then
+                    obj['redirect'] = url_join(url, redirect)
+                end
+
+                db_add('url', obj)
+            end
+        end
+    end
+end

modules/harness/fail.lua

@@ -1,6 +1,5 @@
 -- Description: Test error handling
 -- Version: 0.1.0
--- Source: domains
 -- License: GPL-3.0
 
 function run()

modules/harness/ip.lua

@@ -0,0 +1,15 @@
+-- Description: Show your ip
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function get(url)
+    req = http_request(session, 'GET', url, {})
+    r = http_send(req)
+    info(r['text'])
+end
+
+function run()
+    session = http_mksession()
+    get('https://icanhazip.com')
+    get('https://icanhazptr.com')
+end

modules/harness/keyring.lua

@@ -0,0 +1,9 @@
+-- Description: Request access to keyring
+-- Version: 0.1.0
+-- Keyring-Access: twilio
+-- License: GPL-3.0
+
+function run(arg)
+    keys = keyring('twilio')
+    debug(keys)
+end

modules/harness/keyring2.lua

@@ -0,0 +1,9 @@
+-- Description: Request access to keyring
+-- Version: 0.1.0
+-- Source: keyring:twilio
+-- Keyring-Access: twilio
+-- License: GPL-3.0
+
+function run(arg)
+    info(arg)
+end

modules/harness/options.lua

@@ -0,0 +1,7 @@
+-- Description: Read an option
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info(getopt('hello'))
+end

modules/harness/selftest.lua

@@ -0,0 +1,8 @@
+-- Description: basic selftest
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- nothing to do here
+    info('ohai')
+end

modules/harness/stdin.lua

@@ -0,0 +1,13 @@
+-- Description: Read from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+        info(x)
+    end
+end

modules/harness/tcp-hello.lua

@@ -0,0 +1,18 @@
+-- Description: Send a hello to a server on port 1337
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    addr = getopt('addr')
+    if not addr then
+        return 'addr is not set'
+    end
+
+    -- create connection
+    c = sock_connect(addr, 1337, {})
+    if last_err() then return end
+
+    -- send ohai
+    sock_sendline(c, 'ohai')
+    if last_err() then return end
+end

modules/harness/ttl.lua

@@ -0,0 +1,9 @@
+-- Description: Add an expiring domain
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    db_add_ttl('domain', {
+        value='example.com',
+    }, 30)
+end

modules/harness/tux-img.lua

@@ -0,0 +1,21 @@
+-- Description: Download an image
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+
+    req = http_request(session, 'GET', 'https://www.kernel.org/theme/images/logos/tux.png', {
+        into_blob=true,
+    })
+    r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then
+        return 'http error: ' .. r['status']
+    end
+
+    debug(r)
+    db_add('image', {
+        value=r['blob'],
+    })
+end

sn0int-registry/.gitignore

@@ -1 +1,3 @@
 .env
+# this can be removed after -registry rejoins the workspace
+/target/