Release v0.17.1

seccomp: whitelist ppoll instead of poll on aarch64

.dockerignore

@@ -1,6 +1,7 @@
 target
 Dockerfile
 .dockerignore
+docker-compose.yml
 docker
 docs
 ci

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [kpcyrd]

.travis.yml

@@ -5,18 +5,6 @@ language: rust
 
 matrix:
   include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
   - os: linux
     rust: stable
     env:
@@ -25,6 +13,22 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=docker-registry
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=test
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=common
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=std
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=boxxy
   - os: osx
     rust: stable
     env:
@@ -33,14 +37,18 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=common
+  - os: osx
+    rust: stable
+    env:
+    - BUILD_MODE=std
   #- os: windows
   #  rust: stable
   #  env:
   #  - BUILD_MODE="windows test"
-  - os: windows
-    rust: stable
-    env:
-    - BUILD_MODE="windows common"
+  #- os: windows
+  #  rust: stable
+  #  env:
+  #  - BUILD_MODE="windows common"
 
 before_install:
 - ci/setup.sh "$TRAVIS_OS_NAME"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.10.0"
+version = "0.17.1"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -9,73 +9,77 @@ categories = ["command-line-utilities"]
 readme = "README.md"
 edition = "2018"
 
-[profile.release]
-# skip lto to avoid compiler bug:
-# https://github.com/kpcyrd/sn0int/issues/77
-# https://github.com/rust-lang/rust/issues/58674
-opt-level = 1
-lto = false
-
 [badges]
 travis-ci = { repository = "kpcyrd/sn0int" }
 
 [workspace]
-# rocket is broken and blocks the whole workspace, removing
-#members = ["sn0int-registry/sn0int-common",
-#           "sn0int-registry"]
-members = ["sn0int-registry/sn0int-common"]
+members = ["sn0int-common",
+           "sn0int-registry",
+           "sn0int-std"]
+
+[features]
+sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.5.0", path="sn0int-registry/sn0int-common" }
-rustyline = "3"
+sn0int-common = { version="0.10.0", path="sn0int-common" }
+sn0int-std = { version="0.17.1", path="sn0int-std" }
+rustyline = "6.0"
 log = "0.4"
-env_logger = "0.6"
+env_logger = "0.7"
 hlua-badtouch = "0.4"
-structopt = "0.2"
+structopt = "0.3"
 failure = "0.1"
-rand = "0.6"
+rand = "0.7"
 colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
-publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
+libsqlite3-sys = "0.16.0"
 chrono = { version = "0.4", features = ["serde"] }
-dirs = "1.0"
-url = "1.7"
+dirs = "2.0"
+url = "2.0"
+percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.9"
-base64 = "0.10"
-kuchiki = "0.7.2"
-serde_urlencoded = "0.5"
+chrootable-https = "0.14"
+base64 = "0.11"
+data-encoding = "2.1.2"
+serde_urlencoded = "0.6"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.4"
 ctrlc = "3.1"
-opener = "0.3.0"
+opener = "0.4"
 separator = "0.4"
 maplit = "1.0.1"
-sloppy-rfc4880 = "0.1.2"
+sloppy-rfc4880 = "0.1.5"
 regex = "1.0"
-toml = "0.4"
-maxminddb = "0.13"
-tar = "0.4.17"
-libflate = "0.1.14"
+toml = "0.5"
 threadpool = "1.7"
-x509-parser = "0.4.0"
-der-parser = "1.1.0"
-nom = "4.1.1"
 atty = "0.2"
-bufstream = "0.1.4"
-tokio = "0.1.14"
 semver = "0.9"
+bytes = "0.4"
+bytesize = "1.0"
+ipnetwork = "0.16"
+strum = "0.17"
+strum_macros = "0.17"
+
+digest = "0.8.0"
+md-5 = "0.8.0"
+sha-1 = "0.8.1"
+sha2 = "0.8.0"
+sha3 = "0.8.0"
+hmac = "0.7"
+
+walkdir = "2.2"
+nude = "0.3"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.11"
-nix = "0.13"
+syscallz = "0.12"
+nix = "0.17"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.3.1"
@@ -83,8 +87,5 @@ unveil = "0.2.0"
 
 [dev-dependencies]
 #boxxy = { path = "../boxxy-rs" }
-boxxy = "0.9"
-
-#[patch.crates-io]
-#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }
-#rocket_contrib = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }
+boxxy = "0.11"
+tempfile = "3.0"

Dockerfile

@@ -1,13 +1,14 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

Makefile

@@ -1,21 +1,32 @@
 check:
-	(cd sn0int-registry/sn0int-common; cargo check)
+	(cd sn0int-common; cargo check)
 	(cd sn0int-registry; cargo check)
+	(cd sn0int-std; cargo check)
 	cargo check
 
 force-check:
-	(cd sn0int-registry/sn0int-common; touch src/lib.rs; cargo check)
+	(cd sn0int-common; touch src/lib.rs; cargo check)
 	(cd sn0int-registry; touch src/main.rs; cargo check)
+	(cd sn0int-std; touch src/lib.rs; cargo check)
 	touch src/lib.rs
 	cargo check
 
 test:
-	(cd sn0int-registry/sn0int-common; cargo test)
+	(cd sn0int-common; cargo test)
 	(cd sn0int-registry; cargo test)
-	cargo test
-	cargo test -- --ignored
+	(cd sn0int-std; cargo test)
+	(cd sn0int-std; cargo test -- --ignored)
+	cargo test --lib
+	cargo test --lib -- --ignored
 
 update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt
 	get-iab -v -u http://standards-oui.ieee.org/iab/iab.txt -f data/ieee-iab.txt
 	rm -f data/ieee-*.txt.bak
+
+docs:
+	$(MAKE) -C docs html
+	contrib/html-toc2md.pl README.md docs/_build/html/index.html > README2.md
+	mv README2.md README.md
+
+.PHONY: check force-check test update docs

README.md

@@ -1,4 +1,4 @@
-# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs]
+# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
 [travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
 [travis]:       https://travis-ci.org/kpcyrd/sn0int
@@ -6,67 +6,135 @@
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
 [docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
+[irc-img]:      https://img.shields.io/badge/hackint-%23sn0int-blue.svg
+[irc]:          https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int
+[twitter-img]:  https://img.shields.io/badge/twitter-@sn0int-blue.svg
+[twitter]:      https://twitter.com/sn0int
+[mastodon-img]: https://img.shields.io/badge/mastodon-chaos.social-blue.svg
+[mastodon]:     https://chaos.social/@sn0int
+[registry-img]: https://img.shields.io/website/https/sn0int.com.svg?label=registry
+[registry]:     https://sn0int.com/
 
-sn0int is a semi-automatic OSINT framework and package manager. It was built
-for IT security professionals and bug hunters to gather intelligence about a
-given target or about yourself. sn0int is enumerating attack surface by
-semi-automatically processing public information and mapping the results in a
-unified format for followup investigations.
+sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
+package manager. It was built for IT security professionals and bug hunters to
+gather intelligence about a given target or about yourself. sn0int is
+enumerating attack surface by semi-automatically processing public information
+and mapping the results in a unified format for followup investigations.
+
+[ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
 
 Among other things, sn0int is currently able to:
 
-- Harvest subdomains from certificate transparency logs
-- Harvest subdomains from various passive dns logs
-- Sift through subdomain results for publicly accessible websites
-- Harvest emails from pgp keyservers
-- Enrich ip addresses with ASN and geoip info
-- Harvest subdomains from the wayback machine
+- Harvest subdomains from certificate transparency logs and passive dns
+- Enrich ip addresses with asn and geoip info
+- Harvest emails from pgp keyservers and whois
+- Discover compromised logins in breaches
+- Find somebody's profiles across the internet
+- Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
-- Bruteforce interesting urls
+- Attempt to bypass cloudflare with shodan
+- Harvest data and images from instagram profiles
+- Scan images for nudity
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
-and is fully opensource.  None of the investigations listed above are hardcoded
+and is fully opensource. None of the investigations listed above are hardcoded
 in the source, instead those are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
 you to ship updates for your modules on your own since you don't need to send a
 pull request.
 
-Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
+For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
 [![asciicast](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB.svg)](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB)
 
+## Installation
+
+Archlinux
+
+    pacman -S sn0int
+
+Mac OSX
+
+    brew install sn0int
+
+For everything else please have a look at the [detailed list][1].
+
+[1]: https://sn0int.readthedocs.io/en/latest/install.html
+
 ## Getting started
 
 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
-  - [Debian testing/Debian sid/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-testing-debian-sid-kali)
-  - [Ubuntu/Debian stable](https://sn0int.readthedocs.io/en/latest/install.html#ubuntu-debian-stable)
+  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
   - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
   - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
+  - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
+- [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
+  - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
+    - [Archlinux](https://sn0int.readthedocs.io/en/latest/build.html#archlinux)
+    - [Mac OSX](https://sn0int.readthedocs.io/en/latest/build.html#mac-osx)
+    - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/build.html#debian-ubuntu-kali)
+    - [Alpine](https://sn0int.readthedocs.io/en/latest/build.html#alpine)
+    - [OpenBSD](https://sn0int.readthedocs.io/en/latest/build.html#openbsd)
+    - [Gentoo](https://sn0int.readthedocs.io/en/latest/build.html#gentoo)
+    - [Windows](https://sn0int.readthedocs.io/en/latest/build.html#windows)
+  - [Building](https://sn0int.readthedocs.io/en/latest/build.html#building)
 - [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
   - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
   - [Adding something to scope](https://sn0int.readthedocs.io/en/latest/usage.html#adding-something-to-scope)
   - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
   - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
   - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
-- [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
-  - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
+  - [Autonoscope](https://sn0int.readthedocs.io/en/latest/usage.html#autonoscope)
+    - [Domains](https://sn0int.readthedocs.io/en/latest/usage.html#domains)
+    - [IPs](https://sn0int.readthedocs.io/en/latest/usage.html#ips)
+    - [URLs](https://sn0int.readthedocs.io/en/latest/usage.html#urls)
+- [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
+  - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Publish your repo](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-repo)
   - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
+- [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/structs.html#domains)
+  - [Subdomains](https://sn0int.readthedocs.io/en/latest/structs.html#subdomains)
+  - [IpAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#ipaddrs)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/structs.html#urls)
+  - [Emails](https://sn0int.readthedocs.io/en/latest/structs.html#emails)
+  - [Phonenumbers](https://sn0int.readthedocs.io/en/latest/structs.html#phonenumbers)
+  - [Devices](https://sn0int.readthedocs.io/en/latest/structs.html#devices)
+  - [Networks](https://sn0int.readthedocs.io/en/latest/structs.html#networks)
+  - [Accounts](https://sn0int.readthedocs.io/en/latest/structs.html#accounts)
+  - [Breaches](https://sn0int.readthedocs.io/en/latest/structs.html#breaches)
+  - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
+  - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
+  - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
+  - [CryptoAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#cryptoaddrs)
+  - [Activity](https://sn0int.readthedocs.io/en/latest/structs.html#activity)
+  - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
+    - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
+    - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
+    - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Activity](https://sn0int.readthedocs.io/en/latest/activity.html)
+  - [Anatomy of an event](https://sn0int.readthedocs.io/en/latest/activity.html#anatomy-of-an-event)
+  - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
+  - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
+  - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
   - [Using access keys as source argument](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-as-source-argument)
 - [Configuration](https://sn0int.readthedocs.io/en/latest/config.html)
-  - [Configuring a proxy](https://sn0int.readthedocs.io/en/latest/config.html#configuring-a-proxy)
+  - [\[core\]](https://sn0int.readthedocs.io/en/latest/config.html#core)
+  - [\[namespaces\]](https://sn0int.readthedocs.io/en/latest/config.html#namespaces)
+  - [\[network\]](https://sn0int.readthedocs.io/en/latest/config.html#network)
 - [Sandbox](https://sn0int.readthedocs.io/en/latest/sandbox.html)
   - [Linux](https://sn0int.readthedocs.io/en/latest/sandbox.html#linux)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/sandbox.html#openbsd)
@@ -74,7 +142,15 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
   - [Diagnosing a sandbox failure](https://sn0int.readthedocs.io/en/latest/sandbox.html#diagnosing-a-sandbox-failure)
 - [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
+  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
+  - [base64_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-decode)
+  - [base64_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-encode)
+  - [base64_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-decode)
+  - [base64_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-encode)
+  - [base32_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-decode)
+  - [base32_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-encode)
   - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
+  - [create_blob](https://sn0int.readthedocs.io/en/latest/reference.html#create-blob)
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
@@ -82,27 +158,52 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
   - [error](https://sn0int.readthedocs.io/en/latest/reference.html#error)
-  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
   - [geoip_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#geoip-lookup)
+  - [hex](https://sn0int.readthedocs.io/en/latest/reference.html#hex)
+  - [hmac_md5](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-md5)
+  - [hmac_sha1](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha1)
+  - [hmac_sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-256)
+  - [hmac_sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-512)
+  - [hmac_sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-256)
+  - [hmac_sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-512)
   - [html_select](https://sn0int.readthedocs.io/en/latest/reference.html#html-select)
   - [html_select_list](https://sn0int.readthedocs.io/en/latest/reference.html#html-select-list)
   - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
   - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
   - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [http_fetch](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch)
+  - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
+  - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
+  - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
+  - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [ratelimit_throttle](https://sn0int.readthedocs.io/en/latest/reference.html#ratelimit-throttle)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
+  - [set_err](https://sn0int.readthedocs.io/en/latest/reference.html#set-err)
+  - [sha1](https://sn0int.readthedocs.io/en/latest/reference.html#sha1)
+  - [sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-256)
+  - [sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-512)
+  - [sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-256)
+  - [sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-512)
   - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [sn0int_time](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time)
+  - [sn0int_time_from](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time-from)
+  - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
+  - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
+  - [sock_options](https://sn0int.readthedocs.io/en/latest/reference.html#sock-options)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
   - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
   - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
@@ -116,6 +217,12 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [str_find](https://sn0int.readthedocs.io/en/latest/reference.html#str-find)
+  - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
+  - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
+  - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
   - [url_escape](https://sn0int.readthedocs.io/en/latest/reference.html#url-escape)
@@ -123,7 +230,36 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [url_parse](https://sn0int.readthedocs.io/en/latest/reference.html#url-parse)
   - [url_unescape](https://sn0int.readthedocs.io/en/latest/reference.html#url-unescape)
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
+  - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
+  - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
+  - [ws_connect](https://sn0int.readthedocs.io/en/latest/reference.html#ws-connect)
+  - [ws_options](https://sn0int.readthedocs.io/en/latest/reference.html#ws-options)
+  - [ws_recv_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-text)
+  - [ws_recv_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-binary)
+  - [ws_recv_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-json)
+  - [ws_send_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-text)
+  - [ws_send_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-binary)
+  - [ws_send_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-json)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
+  - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
+  - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)
+
+## Rationale
+
+This tool was written for companies to help them understand their attack
+surface from a blackbox point of view. It's often difficult to understand that
+something is easier to discover than some people assume, putting them at risk
+of false security.
+
+It's also designed to be useful for red team assessments and bug bounties,
+which also help companies to identify weaknesses that could result in a
+compromise.
+
+Some functionality was written to do the same thing for individuals to raise
+awareness about personal attack surface, privacy and how much data is publicly
+available. These issues are often out of scope in bug bounties and sometimes by
+design. We believe that blaming the user is the wrong approach and these issues
+should be addressed at the root cause by the people designing those systems.
 
 ## License
 

ci/integration.py

@@ -63,6 +63,7 @@ def main(tempdir, binary):
     print('[*] testing db for subdomains')
     subdomains = sn0int_select(tempdir, binary, ['subdomains'])
     assert {x['value'] for x in subdomains} == {
+        'example.com',
         'www.example.com',
         'm.example.com',
         'dev.example.com',
@@ -91,6 +92,8 @@ def main(tempdir, binary):
     print('[*] testing db for urls')
     urls = sn0int_select(tempdir, binary, ['urls'])
     assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
         ('http://www.example.com/', 200),
         ('https://www.example.com/', 200),
     }

ci/run.sh

@@ -6,21 +6,21 @@ case "$1" in
         cargo build --verbose --examples
         ;;
     test)
-        ci/run.sh build
-        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
-             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
         cargo test --verbose
         cargo test --verbose -- --ignored
         ;;
     common)
-        cd sn0int-registry/sn0int-common
+        cd sn0int-common
         cargo test --verbose
         ;;
+    std)
+        cd sn0int-std
+        cargo test --verbose
+        cargo test --verbose -- --ignored
+        ;;
     windows)
-        export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
-        ci/run.sh "$2"
+        cargo build --verbose --features=sqlite-bundled
+        cargo build --verbose --examples --features=sqlite-bundled
         ;;
     boxxy)
         cargo build --verbose --examples
@@ -35,7 +35,7 @@ case "$1" in
         docker run --rm sn0int --help
         ;;
     docker-registry)
-        docker build -t sn0int-registry sn0int-registry/
+        docker build -t sn0int-registry -f sn0int-registry/Dockerfile .
         docker images
         ;;
 esac

ci/setup.sh

@@ -5,9 +5,4 @@ case "$1" in
         sudo apt update
         sudo apt install libsqlite3-dev libseccomp-dev
         ;;
-    windows)
-        curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-        7z e sqlite3.zip -y
-        "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
-        ;;
 esac

contrib/html-toc2md.pl

@@ -0,0 +1,39 @@
+#!/usr/bin/env perl
+use strict; use warnings;
+
+my ($readme, $toc) = @ARGV;
+defined $readme or die 'missing readme path';
+defined $toc or die 'missing toc path';
+
+open(my $r, "<$readme") or die 'failed to open readme';
+open(my $t, "<$toc") or die 'failed to open toc';
+
+my $re = qr/^\s*- \[.+\]\(https:\/\/sn0int.readthedocs.io\/en\/.+\)$/;
+
+# pass through start of readme
+while (<$r>) {
+    last if ($_ =~ $re);
+    print $_;
+}
+
+# skip toc
+while (<$r>) {
+    last unless ($_ =~ $re);
+}
+
+# generate new toc
+while (my $line = <$t>) {
+    if ($line =~ /toctree-l(\d).*href="([^"]+)">(.+)<\/a/) {
+        my $space = int($1)-1;
+        my $section = $2;
+        my $label = $3;
+        $label =~ s/([\[\]])/\\$1/g;
+        print " " x ($space*2), "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
+    }
+}
+print;
+
+# pass through end of readme
+while (<$r>) {
+    print $_;
+}

contrib/html-toc2md.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-perl -n -e '/toctree-l(\d).*href="([^"]+)">(.+)<\/a/ && print $1==2?"  ":"", "- [$3](https://sn0int.readthedocs.io/en/latest/$2)\n"' < docs/_build/html/index.html

docs/activity.rst

@@ -0,0 +1,166 @@
+Activity
+========
+
+So far we've learned about regular `structs <structs.html>`_, but activity is
+special.
+
+Activity is an event tied to a specific time and topic and has a small amount
+of data piggybacked to it.
+
+Anatomy of an event
+-------------------
+
+``topic``
+    This is some freestyle text used to group events to a specific topic. This
+    must not conflict with other modules unless there's a very good reason.
+
+    The topic should look like ``kpcyrd/example:something``, with ``something``
+    being a meaningful unique identifier for whatever is generating these
+    events, like a mac address or an account name/id.
+
+    The rules around this might become stricter in the future.
+``time``
+    The most important part of the event: The time and date it happened.
+``initial``
+    This value can not be set but might be present in sn0int output. See `Querying events`_.
+``uniq`` (optional)
+    This is an optional feature to deduplicate events. Assuming you're
+    importing posts by an account, you wouldn't want to store a new event for
+    each post you already imported. If you set this field to the technical post
+    id then sn0int would skip the event if it already has an event with the
+    same ``topic`` and ``uniq`` combination to avoid inserting duplicates.
+``latitude`` (optional)
+    Latitude - if you can tie the event to a specific location.
+``longitude`` (optional)
+    Longitude - if you can tie the event to a specific location.
+``radius`` (optional)
+    The location radius in meters. If the position you got has a precision of
+    100 meters set this value to ``100``.
+``content``
+    Arbitrary data that you want to attach to the event. This doesn't need to
+    be a string and can be an arbitrary object that is then stored as json
+    string.
+
+Logging events
+--------------
+
+An ``activity`` event can be logged with ``db_activity``:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Logging an event that has a location attached could look like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        latitude=40.726662,
+        longitude=-74.036677,
+        radius=50,
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Making sure an event is not logged twice can be done with ``uniq``:
+
+.. code-block:: lua
+
+    -- create the first event
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this does nothing because we already have an event with this topic+uniq combination
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this creates a new event because uniq is different
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+    -- this also creates a new event because topic is different
+    db_activity({
+        topic='harness/activity-ping:something-else',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+Querying events
+---------------
+
+There is a commandline interface that can be used to query all events we've
+logged. To get everything (sorted by time)::
+
+    sn0int activity
+
+To limit the output to a specific topic::
+
+    sn0int activity -t harness/activity-ping:dummy
+
+To limit it to a specific time frame::
+
+    # everything since
+    sn0int activity --since 2020-01-13T04:20:00
+    # everything until
+    sn0int activity --until 2020-01-13T04:20:00
+    # both
+    sn0int activity --since yesterday --until today
+
+When using ``--since`` you might also want to know the previous state and use
+it as an initial value. Consider this example::
+
+    2020-01-13 14:30:00 # user goes offline
+    2020-01-13 23:59:00 # user goes online
+    2020-01-14 09:30:00 # user goes idle
+    2020-01-14 14:20:00 # user goes offline
+
+If we're running a query like ``sn0int activity --since 2020-01-14T00:00:00``
+the program consuming the output wouldn't know that the user is initially
+online because we're only getting this data::
+
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+We can tweak this with ``sn0int activity --initial --since
+2020-01-14T00:00:00`` to include one more event that we only use to populate
+the intial state::
+
+    {"id":7,"initial":true,"topic":"foo/bar:asdf","time":"2020-01-13T23:59:00","content":{"state":"online"}}
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+Visualization
+-------------
+
+There is no visualization built in, there may be external frontends for this in
+the future. You're very welcome to write one!

docs/build.rst

@@ -0,0 +1,80 @@
+Build from source
+=================
+
+It's generally recommended to `install a package <install.html>`_ if available.
+This section is about building the binary from git.
+
+Install dependencies
+--------------------
+
+You need a recent rust compiler. It's usually recommended to install a rust
+compiler with `rustup <https://rustup.rs/>`_, but if you're system ships the
+most recent compiler in a package that works too. Note that some systems aren't
+fully supported by rustup (like OpenBSD and alpine) and you need to install
+rust from a package in that case.
+
+Archlinux
+~~~~~~~~~
+
+.. code-block:: bash
+
+    $ pacman -S geoip2-database libseccomp publicsuffix-list sqlite
+
+Mac OSX
+~~~~~~~
+
+None.
+
+Debian/Ubuntu/Kali
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+
+.. warning::
+   On a debian based system make sure you've installed rust with rustup.
+
+Alpine
+~~~~~~
+
+.. code-block:: bash
+
+    $ apk add sqlite-dev libseccomp-dev
+
+OpenBSD
+~~~~~~~
+
+.. code-block:: bash
+
+    $ pkg_add sqlite3 geolite2-city geolite2-asn
+
+Gentoo
+~~~~~~
+
+.. code-block:: bash
+
+    emerge --ask sys-libs/libseccomp dev-db/sqlite
+
+Windows
+~~~~~~~
+
+You don't need to install any dependencies on windows, but you need to use a
+different build command in the next section.
+
+Building
+--------
+
+After all dependencies have been installed, simply build the binary:
+
+.. code-block:: bash
+
+    $ cargo build --release
+
+Note that you need a different command on windows:
+
+.. code-block:: bash
+
+    $ cargo build --release --features=sqlite-bundled
+
+After the build finished the binary is located at ``target/release/sn0int``.

docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'sn0int'
-copyright = '2018, kpcyrd'
+copyright = '2018-2020, kpcyrd'
 author = 'kpcyrd'
 
 # The short X.Y version

docs/config.rst

@@ -1,17 +1,54 @@
 Configuration
 =============
 
-This file documents the config file at ``~/.config/sn0int.toml``. By default
-this file does not exist and a default configuration is used instead.
+This section documents the config file. By default this file does not exist and
+a default configuration is used instead.
 
-Configuring a proxy
--------------------
+Linux/BSD
+    ``~/.config/sn0int.toml``
+
+OSX
+    ``~/Library/Preferences/sn0int.toml``
+
+Windows
+    ``%APPDATA%/sn0int.toml``
+
+[core]
+------
+
+``registry``
+    Configure the registry you want to use. Defaults to ``https://sn0int.com``.
+``no-autoupdate``
+    sn0int is going to check if your modules are outdated during startout once
+    a week. Set this option to ``true`` to disable this.
+
+[namespaces]
+------------------
+
+By default sn0int modules are assumed to be installed from the registry. You
+may want to keep a local directory with private modules, especially during
+development. You can configure a folder that contains modules that aren't
+managed by sn0int by adding a namespace section to the config file::
+
+    [namespaces]
+    foo = "/opt/sn0int/foo"
+    bar = "~/repos/a/b/c/sn0int-modules"
+
+This is going to load modules from these two folders and register them in the
+``foo`` and ``bar`` namespace.
+
+Note that sn0int is also going to assume that symlinks in
+``~/.local/share/sn0int/modules`` and folders containing a ``.git`` folder are
+externally managed.
+
+[network]
+---------
 
 To enable a proxy, add the following to your config file::
 
     [network]
     proxy = "127.0.0.1:9050"
 
-This forces everything through tor and restricts all other functions that
-depend on the network. For example the ``dns`` function is fully disabled if a
-proxy is configured.
+This forces everything through tor (or any other socks5 proxy) and restricts
+all other functions that depend on the network. For example the ``dns``
+function is fully disabled if a proxy is configured.

docs/database.rst

@@ -46,6 +46,45 @@ triggered and an db_update is performed instead.
    removed from scope with ``noscope``. Everytime you use ``db_add`` you need
    to make sure that the ID that has been returned is not ``nil``.
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add_ttl('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_update
 ---------
 

docs/index.rst

@@ -36,10 +36,12 @@ Getting Started
    :glob:
 
    install
+   build
    usage
    scripting
    database
    structs
+   activity
    keyring
    config
    sandbox

docs/install.rst

@@ -1,7 +1,9 @@
 Installation
 ============
 
-If available, please prefer the package shipped by your linux distribution.
+If available, please prefer the package shipped by operating system. If your
+operating system has a package but you're running on older version, please use
+the `build from source <build.html>`_ instructions instead.
 
 Archlinux
 ---------
@@ -17,27 +19,16 @@ Mac OSX
 
     $ brew install sn0int
 
-Debian testing/Debian sid/Kali
-------------------------------
+Debian/Ubuntu/Kali
+------------------
 
 Note that debian `doesn't ship the geoip2-database
 <https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
 download them automatically during the first run.
 
-.. code-block:: bash
-
-    $ apt install build-essential cargo libsqlite3-dev libseccomp-dev publicsuffix
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
-
-Ubuntu/Debian stable
---------------------
-
-cargo in the repos is too old and the build is `going to fail
-<https://github.com/kpcyrd/sn0int/issues/68>`_. You should either install the
-most recent rust version with `rustup <https://rustup.rs/>`_ or use the docker
-instructions instead.
+Using rust+cargo from the repos might work for you, but we only officially
+support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
+at the docker image as an alternative.
 
 .. code-block:: bash
 
@@ -51,39 +42,40 @@ Docker
 
 .. code-block:: bash
 
-    $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
+    $ docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
 
 Alpine
 ------
 
 .. code-block:: bash
 
-    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ apk add sn0int
 
 OpenBSD
 -------
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ pkg_add sn0int
+
+Gentoo
+------
+
+.. code-block:: bash
+
+    layman -f -o https://raw.githubusercontent.com/kpcyrd/overlay/master/overlay.xml -a kpcyrd-overlay
+    emerge --ask net-analyzer/sn0int
 
 Windows
 -------
 
-This is not recommended and only passively maintained. Please prefer linux in a virtual machine if needed.
+This is not recommended and only passively maintained. Please prefer linux in a
+virtual machine if needed.
+
+Make sure rust is installed and setup.
 
 .. code-block:: bash
 
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-    $ 7z e sqlite3.zip -y
-    $ "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
-    $ export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
-    $ cargo install -f --path .
+    $ cargo install -f --path . --features=sqlite-bundled

docs/keyring.rst

@@ -52,12 +52,16 @@ If the user granted us access to those keys we can read them with ``keyring``:
 .. code-block:: lua
 
     creds = keyring('aws')
-    print(creds[1]['accesskey'])
-    print(creds[1]['secretkey'])
+    debug(creds[1]['access_key'])
+    debug(creds[1]['secret_key'])
 
 This returns a list of all keys in that namespace. Any empty list is returned
 if the user doesn't have any keys in that namespace.
 
+If you want to allow the user to select a specific script you can introduce an
+option that is set by the user and then filter ``creds`` until the
+``access_key`` matches.
+
 Using access keys as source argument
 ------------------------------------
 

docs/reference.rst

@@ -1,6 +1,87 @@
 Function reference
 ==================
 
+asn_lookup
+----------
+
+Run an ASN lookup for a given ip address. The function returns ``asn`` and
+``as_org``. This function may fail.
+
+.. code-block:: lua
+
+    lookup = asn_lookup('1.1.1.1')
+    if last_err() then return end
+
+base64_decode
+-------------
+
+Decode a base64 string with the default alphabet+padding.
+
+.. code-block:: lua
+
+    base64_decode("ww==")
+
+base64_encode
+-------------
+
+Encode a binary array with base64 and the default alphabet+padding.
+
+.. code-block:: lua
+
+    base64_encode("\x00\xff")
+
+base64_custom_decode
+--------------------
+
+Decode a base64 string with custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- base64
+    base64_custom_decode('b2hhaQ==', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '=')
+    -- base64 no padding
+    base64_custom_decode('b2hhaQ', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '')
+    -- base64 url safe
+    base64_custom_decode('b2hhaQ==', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', '=')
+
+base64_custom_encode
+--------------------
+
+Encode a binary array with base64 and custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- base64
+    base64_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '=')
+    -- base64 no padding
+    base64_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '')
+    -- base64 url safe
+    base64_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', '=')
+
+base32_custom_decode
+--------------------
+
+Decode a base32 string with custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- rfc-4648 base32
+    base32_custom_decode('N5UGC2I=', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567', '=')
+    -- z-base-32
+    base32_custom_decode('p7wgn4e', 'ybndrfg8ejkmcpqxot1uwisza345h769', '')
+
+base32_custom_encode
+--------------------
+
+Encode a binary array with base32 and custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- rfc-4648 base32
+    x = base32_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567', '=')
+    -- z-base-32
+    x = base32_custom_encode('ohai', 'ybndrfg8ejkmcpqxot1uwisza345h769', '')
+
 clear_err
 ---------
 
@@ -13,6 +94,18 @@ Clear the last recorded error from the internal state. See also last_err_.
         clear_err()
     end
 
+create_blob
+-----------
+
+Push a byte array into persistent blob storage. This allows passing those bytes
+to functions operating on blob storage. Returns a blob identifier that is
+deterministic based on the blob content. Blobs are immutable.
+
+.. code-block:: lua
+
+    blob = create_blob("some bytes")
+    debug(blob)
+
 datetime
 --------
 
@@ -23,6 +116,14 @@ for ``DATETIME`` database fields.
 
     now = datetime()
 
+.. note::
+    This format is sn0int specific, to get the current time for scripting use
+    time_unix_ instead.
+
+.. warning::
+    This function is going to be deprecated at some point. Prefer sn0int_time_
+    for new scripts.
+
 db_add
 ------
 
@@ -47,11 +148,33 @@ ttl.
 .. code-block:: lua
 
     -- this link is valid for 2min
-    domain_id = db_add('network-device', {
+    domain_id = db_add_ttl('network-device', {
         network_id=1,
         device_id=13,
     }, 120)
 
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_select
 ---------
 
@@ -123,17 +246,6 @@ Log an error to the terminal.
 
     error('ohai')
 
-asn_lookup
-----------
-
-Run an ASN lookup for a given ip address. The function returns ``asn`` and
-``as_org``. This function may fail.
-
-.. code-block:: lua
-
-    lookup = asn_lookup('1.1.1.1')
-    if last_err() then return end
-
 geoip_lookup
 ------------
 
@@ -154,6 +266,69 @@ This function may fail.
     lookup = geoip_lookup('1.1.1.1')
     if last_err() then return end
 
+hex
+---
+
+Hex encode a list of bytes.
+
+.. code-block:: lua
+
+    hex("\x6F\x68\x61\x69\x0A\x00")
+
+hmac_md5
+--------
+
+Calculate an hmac with md5. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_md5("secret", "my authenticated message")
+
+hmac_sha1
+---------
+
+Calculate an hmac with sha1. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha1("secret", "my authenticated message")
+
+hmac_sha2_256
+-------------
+
+Calculate an hmac with sha2_256. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha2_256("secret", "my authenticated message")
+
+hmac_sha2_512
+-------------
+
+Calculate an hmac with sha2_512. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha2_512("secret", "my authenticated message")
+
+hmac_sha3_256
+-------------
+
+Calculate an hmac with sha3_256. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha3_256("secret", "my authenticated message")
+
+hmac_sha3_512
+-------------
+
+Calculate an hmac with sha3_512. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha3_512("secret", "my authenticated message")
+
 html_select
 -----------
 
@@ -210,6 +385,12 @@ options are set. The following options are available:
   The request body that should be form encoded.
 ``body``
   The raw request body as string.
+``into_blob``
+  If true, the response body is stored in blob storage and a blob reference is
+  returned as ``blob`` instead of the full body.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
 
 This function may fail.
 
@@ -223,7 +404,7 @@ This function may fail.
     })
     resp = http_send(req)
     if last_err() then return end
-    if resp["status"] ~= 200 then return "invalid status code" end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
 
 http_send
 ---------
@@ -237,6 +418,9 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``blob``
+  If ``into_blob`` was enabled for the request the body is downloaded into blob
+  storage with a reference to the body in this field.
 
 .. code-block:: lua
 
@@ -248,7 +432,81 @@ the following keys:
     })
     resp = http_send(req)
     if last_err() then return end
-    if resp["status"] ~= 200 then return "invalid status code" end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+
+http_fetch
+----------
+
+This does an http_send_ and also automatically validate the status code.
+
+.. note::
+   You almost always want this when setting the ``into_blob`` option since this
+   function validates the status code *before* inserting the response body into
+   blob storage.
+
+.. code-block:: lua
+
+    -- short form
+    data = http_fetch(req)
+    if last_err() then return end
+
+    -- long form
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+
+http_fetch_json
+---------------
+
+Identical to http_fetch_ but also automatically parses the response body as json.
+
+.. code-block:: lua
+
+    -- short form
+    data = http_fetch_json(req)
+    if last_err() then return end
+
+    -- long form
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+    data = json_decode(resp['text'])
+    if last_err() then return end
+
+img_load
+--------
+
+Attempt to decode a blob as an image and return some basic metadata like the
+mime type, height and width.
+
+.. code-block:: lua
+
+    img = img_load(blob)
+    if last_err() then return end
+    debug(img)
+
+img_exif
+--------
+
+Extract exif metadata from an image.
+
+.. code-block:: lua
+
+    exif = img_exif(blob)
+    if last_err() then return end
+    debug(exif)
+
+img_nudity
+----------
+
+Classify an image for nudity. The score goes from 0 to 2. A score above 1 means
+nudity has been detected.
+
+.. code-block:: lua
+
+    nudity = img_nudity(blob)
+    if last_err() then return end
+    debug(nudity)
 
 info
 ----
@@ -318,6 +576,15 @@ otherwise.
         return
     end
 
+md5
+---
+
+Hash a byte array with md5 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(md5("\x00\xff"))
+
 pgp_pubkey
 ----------
 
@@ -326,7 +593,8 @@ Same as pgp_pubkey_armored_, but without the unarmor step.
 pgp_pubkey_armored
 ------------------
 
-Extract uids out of a rfc 4880 pgp public key. This function may fail.
+Extract ``uids``, ``sigs`` and the ``fingerprint`` out of an rfc 4880 pgp
+public key. This function may fail.
 
 .. code-block:: lua
 
@@ -397,6 +665,22 @@ Returns the parent domain according to the public suffix list. For
     domain = psl_domain_from_dns_name('www.a.b.c.d.example.co.uk')
     print(domain == 'example.co.uk')
 
+ratelimit_throttle
+------------------
+
+Create a ratelimit that can only be passed x times every y milliseconds. This
+limit is global for a single ``run`` and also works with threads.
+
+.. code-block:: lua
+
+    -- allow this to pass every 250ms
+    ratelimit_throttle('foo', 1, 250)
+    -- allow this to pass not more than 4 times per second
+    ratelimit_throttle('foo', 4, 1000)
+
+This is useful if you need to coordinate your executions to stay below a
+certain request threshold.
+
 regex_find
 ----------
 
@@ -430,6 +714,86 @@ Same as regex_find_, but returns all matches.
     print(m[3][1] == 'ef')
     print(m[3][2] == 'f')
 
+semver_match
+------------
+
+Compare a version to a version requirement. This can be used with
+sn0int_version_ to test for certain features or behavior.
+
+.. code-block:: lua
+
+    semver_match('=0.11.2', sn0int_version())
+    semver_match('>0.11.2', sn0int_version())
+    semver_match('<0.11.2', sn0int_version())
+    semver_match('~0.11.2', sn0int_version())
+    semver_match('^0.11.2', sn0int_version())
+    semver_match('0.11.2', sn0int_version()) -- synonym for ^0.11.2
+    semver_match('<=0.11.2', sn0int_version())
+    semver_match('>=0.11.2', sn0int_version())
+    semver_match('>=0.4.0, <=0.10.0', sn0int_version())
+
+set_err
+-------
+
+Manipulate the global error object. If you want to exit the main ``run``
+function with an error you can simply return a string, but those are difficult
+to propagate through functions. ``set_err`` specifically assigns an error to
+the global error object that are also used by all other rust functions.
+
+.. code-block:: lua
+
+    function foo()
+        set_err("something failed")
+    end
+
+    foo()
+    if last_err() then return end
+
+sha1
+----
+
+Hash a byte array with sha1 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha1("\x00\xff"))
+
+sha2_256
+--------
+
+Hash a byte array with sha2_256 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha2_256("\x00\xff"))
+
+sha2_512
+--------
+
+Hash a byte array with sha2_512 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha2_512("\x00\xff"))
+
+sha3_256
+--------
+
+Hash a byte array with sha3_256 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha3_256("\x00\xff"))
+
+sha3_512
+--------
+
+Hash a byte array with sha3_512 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha3_512("\x00\xff"))
+
 sleep
 -----
 
@@ -440,14 +804,112 @@ only used for debugging.
 
     sleep(1)
 
+sn0int_time
+-----------
+
+Return current time in UTC. This function is suitable to determine datetimes
+for ``DATETIME`` database fields.
+
+.. code-block:: lua
+
+    now = sn0int_time()
+
+.. note::
+    This format is sn0int specific, to get the current time for scripting use
+    time_unix_ instead.
+
+sn0int_time_from
+----------------
+
+Identical to sn0int_time_ but uses a unix timestamp in seconds instead of the
+current time. This function is compatible with time_unix_ and strptime_.
+
+.. code-block:: lua
+
+    time = sn0int_time_from(1567931337)
+
+sn0int_version
+--------------
+
+Get the current sn0int version string. This can be used with semver_match_ to
+test for certain features or behavior.
+
+.. code-block:: lua
+
+    info(sn0int_version())
+
 sock_connect
 ------------
 
 Create a tcp connection.
 
+The following options are available:
+
+``tls``
+  Set to true to enable tls (certificates are validated)
+``sni_value``
+  Instead of the host argument, use a custom string for the sni extension.
+``disable_tls_verify``
+  **Danger**: disable tls verification. This disables all security on the
+  connection. Note that sn0int is still rather strict, you're going to run into
+  issues if you need support for insecure ciphers.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
 .. code-block:: lua
 
-    sock = sock_connect("127.0.0.1", 1337)
+    sock = sock_connect("127.0.0.1", 1337, {
+        tls=true,
+    })
+
+sock_upgrade_tls
+----------------
+
+Take an existing tcp connection and start a tls handshake. The options are the
+same as sock_connect_ but the ``tls`` value is always assumed to be true.
+
+The sni value needs to be set specifically, otherwise the sni extension is
+disabled.
+
+Using this function specifically returns some extra information that is
+discarded when using sock_connect_ directly with ``tls=true``.
+
+.. code-block:: lua
+
+    sock = sock_connect("127.0.0.1", 1337, {})
+    if last_err() then return end
+
+    tls = sock_upgrade_tls(sock, {
+        sni_value='example.com',
+    })
+    if last_err() then return end
+
+    info(tls)
+
+sock_options
+------------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock_options(sock, {
+        read_timeout=3,
+    })
 
 sock_send
 ---------
@@ -571,6 +1033,76 @@ Read a line from stdin. The final newline is not removed.
 .. note::
    This only works with `sn0int run --stdin`.
 
+.. TODO: add stdin_read_line and deprecate stdin_readline
+
+stdin_read_to_end
+-----------------
+
+Read stdin until EOF as a utf-8 string.
+
+.. code-block:: lua
+
+    stdin_read_to_end()
+
+.. note::
+   This only works with `sn0int run --stdin`.
+
+str_find
+--------
+
+Returns the byte index of the first character that matches the pattern. This is
+explicitly a literal match instead of a lua pattern.
+
+If no match is found, returns ``nil``.
+
+.. code-block:: lua
+
+    x = str_find('asdf', 'sd')
+    print(x == 2)
+
+str_replace
+-----------
+
+Replaces all matches of a pattern in a string. This is explicitly a literal
+match instead of a lua pattern.
+
+If no match is found, an unmodified copy is returned.
+
+.. code-block:: lua
+
+    x = str_replace('this is old', 'old', 'new')
+    print(x == 'this is new')
+
+strftime
+--------
+
+Format a timestamp generated with time_unix_ into a date, see `strftime rules`_.
+
+.. code-block:: lua
+
+    t = strftime('%d/%m/%Y %H:%M', 1558584994)
+
+strptime
+--------
+
+Parse a date into a unix timestamp, see `strftime rules`_.
+
+.. code-block:: lua
+
+    t = strptime('%d/%m/%Y %H:%M', '23/05/2019 04:16')
+
+.. _strftime rules: https://docs.rs/chrono/0.4.6/chrono/format/strftime/index.html
+
+time_unix
+---------
+
+Get the current time as seconds since ``January 1, 1970 0:00:00 UTC``, also
+known as UNIX timestamp. This timestamp can be formated using strftime_.
+
+.. code-block:: lua
+
+    now = time_unix()
+
 url_decode
 ----------
 
@@ -660,6 +1192,132 @@ Decodes a list of bytes/numbers into a string. This function might fail.
     if last_err() then return end
     print(x == 'AAAA')
 
+warn
+----
+
+Log a warning to the terminal.
+
+.. code-block:: lua
+
+    warn('ohai')
+
+warn_once
+---------
+
+Log a warning to the terminal once. This can be used to print a warning to the
+user without printing the same warning for each struct we're processing during
+a ``run`` execution.
+
+.. code-block:: lua
+
+    warn_once('ohai')
+    warn_once('ohai')
+
+ws_connect
+----------
+
+Create a websocket connection. The url format is ``ws://example.com/asdf``,
+``wss://`` is also supported.
+
+The following options are available:
+
+``headers``
+  A map of additional headers that should be set for the request.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock = ws_connect("wss://example.com/asdf", {})
+
+ws_options
+----------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    ws_options(sock, {
+        read_timeout=3,
+    })
+
+ws_recv_text
+------------
+
+Wait until the server sends a text frame. A binary frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_text(sock)
+
+ws_recv_binary
+--------------
+
+Wait until the server sends a binary frame. A text frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_binary(sock)
+
+ws_recv_json
+------------
+
+Identical to ws_send_text_ but automatically runs json_decode_ on the
+response.
+
+.. code-block:: lua
+
+    msg = ws_recv_json(sock)
+
+ws_send_text
+------------
+
+Send a text frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, "ohai!")
+
+ws_send_binary
+--------------
+
+Send a binary frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_binary(sock, "\x00\x01\x02")
+
+ws_send_json
+------------
+
+Encode the object as json string and send it as a text frame on the websocket
+connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, {
+        foo="ohai!",
+        x={
+            y={1,3,3,7},
+        },
+    })
+
 x509_parse_pem
 --------------
 
@@ -694,3 +1352,35 @@ Parse a pem encoded certificate. This function might fail.
     ]])
     if last_err() then return end
     print(x)
+
+xml_decode
+----------
+
+Decode a lua value from an xml document.
+
+.. code-block:: lua
+
+    x = xml_decode('<body><foo fizz="buzz">bar</foo></body>')
+    if last_err() then return end
+
+    body = x['children'][1]
+    foo = body['children'][1]
+
+    print(foo['attrs']['fizz'])
+    print(foo['text'])
+
+xml_named
+---------
+
+Get a named child element from a parent element.
+
+.. code-block:: lua
+
+    x = xml_decode('<body><foo fizz="buzz">bar</foo></body>')
+    if last_err() then return end
+
+    body = x['children'][1]
+    foo = xml_named(body, 'foo')
+    if foo ~= nil then
+        print(foo)
+    end

docs/scripting.rst

@@ -1,11 +1,11 @@
-Scripting
-=========
+Writing your first module
+=========================
 
 Scripting is the core feature in sn0int. It's not strictly required, but if you
 want to write your own modules, this section is for you.
 
-Write your first module
------------------------
+Creating a repository
+---------------------
 
 It's highly recommended to use a VCS for development, so let's start by setting
 that up. We're going to assume you store your repos in ``~/repos`` but you're
@@ -13,7 +13,25 @@ free to change that to something else::
 
     $ git init ~/repos/sn0int-modules
     $ cd ~/repos/sn0int-modules
-    $ ln -s "$PWD" ~/.local/share/sn0int/modules/$YOUR_GITHUB_NAME
+
+.. note::
+   If you're using github you can also create a repo from the `module repo
+   template`_.
+
+.. _module repo template: https://github.com/sn0int/sn0int-modules
+
+We need to add this folder to the sn0int config file so it's correctly detected
+when starting sn0int. Open the `config file <config.html>`_ in your prefered
+editor. Note that the file does not exist by default and the path is different
+depending on your operating system. On linux you would open the config file
+with::
+
+    $ vim ~/.config/sn0int.toml
+
+Add the following::
+
+    [namespaces]
+    your_github_name = "~/repos/sn0int-modules"
 
 Every module we're adding to ``~/repos/sn0int-modules`` is now going to be
 picked up by sn0int.
@@ -78,21 +96,26 @@ database.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
-        print(subdomain)
+        info(subdomain)
     end
 
-Combined with the header we wrote previously we can already execute this
-module. Make sure you've added a domain to scope with ``add domain
-example.com``, save your file and run it like this::
+This is already enough to execute it. Make sure you've added a domain to scope
+with ``add domain example.com``, save your file and run it like this::
 
     sn0int run -f ./first.lua
 
-We should see some output by our print function.
+We should see some output by our info function.
 
 .. note::
-   ``print`` is useful for development but must be removed before publishing.
+   ``info`` is useful for development but you usually want your module to run
+   quietly, so before publishing either remove it or replace it with ``debug``.
 
 Next, we want to actually resolve that name, we're going to use the ``dns``
 function for that. This function takes a name and a query type and returns a
@@ -102,6 +125,11 @@ truth-y.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
@@ -110,16 +138,22 @@ truth-y.
         })
         if last_err() then return end
 
-        print(records)
+        info(records)
     end
 
 If you run your module again you're going to see some output, either
 ``{"answers":[somedata],"error":null}`` or
-``{"answers":[],"error":"NXDomain"}``. We decide that we add the subdomain to
-our scope and set it to resolvable if ``error`` is ``nil``.
+``{"answers":[],"error":"NXDomain"}``. If the dns reply doesn't indicate an
+error this means the subdomain exists and we can add it to our database with
+``resolvable`` being set to ``true``.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
@@ -140,41 +174,20 @@ our scope and set it to resolvable if ``error`` is ``nil``.
 .. hint::
    See the database section to understand how the database works in detail.
 
-If we execute our module one more time it's going to log that it discovered a
-subdomain, if it doesn't, try adding more domains to scope. Note that this only
-happens the first time. Modules that don't discover anything or don't discover
-anything new exit silently.
-
-After putting everything together, our final module looks like this:
-
-.. code-block:: lua
-
-    -- Description: ohai wurld
-    -- Version: 0.1.0
-    -- Source: domains
-    -- License: GPL-3.0
-
-    function run(arg)
-        subdomain = 'www.' .. arg['value']
-
-        records = dns(subdomain, {
-            record='A'
-        })
-        if last_err() then return end
-
-        if records['success'] ~= nil then
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=subdomain,
-                resolvable=true,
-            })
-        end
-    end
+If we execute our finished module one more time it's going to log that it
+discovered a subdomain, if it doesn't, try adding more domains to scope. Note
+that this only happens the first time. Modules that don't discover anything or
+don't discover anything new exit silently.
 
 There's still some room for improvement, for example, since we already resolved
 that record, we could also add the ip address to the scope and link it to the
 subdomain we added.
 
+.. hint::
+   For debugging purposes you can increase the verbosity with ``sn0int run -v``
+   so database operations are logged even if nothing was changed, or with
+   ``sn0int run -vv`` to enable ``debug()`` output.
+
 Publish your module
 -------------------
 
@@ -193,6 +206,24 @@ Afterwards publish your module with::
 
     sn0int publish ./first.lua
 
+Please also make sure you publish your repository to github so other people can
+submit pull requests. The recommended repository location is::
+
+    https://github.com/<your-username>/sn0int-modules
+
+Publish your repo
+-----------------
+
+It is highly recommended to publish your repository on github so people can
+file issues and pull requests for your module. If you've been following along
+with the github template you can simply commit your changes and push them.
+
+Your repository would look like one of these:
+
+- https://github.com/kpcyrd/sn0int-modules
+- https://github.com/ysf/sn0int-modules
+- https://github.com/cybiere/sn0int-modules
+
 Reading data from stdin
 -----------------------
 

docs/structs.rst

@@ -86,6 +86,8 @@ Emails
 
 ``value``
     The email address.
+``displayname``
+    The display name of a given email address: ``this is the name <foo@example.com>``.
 ``valid``
     Whether that email address is valid or has been disabled.
 
@@ -143,6 +145,8 @@ connected to.
     Latitude of the networks location.
 ``longitude``
     Longitude of the networks location.
+``description``
+    A human readable description in case the value is a technical identifier.
 
 Accounts
 --------
@@ -164,6 +168,12 @@ A users account or profile on a webservice, like github or instagram.
     The url of the public profile if available.
 ``last_seen``
     The last time this account has been active/online.
+``birthday``
+    The users birthday set on the account.
+``phonenumber``
+    The phonenumber associated with the account.
+``profile_pic``
+    The blob identifier of the users current profile picture.
 
 Breaches
 --------
@@ -174,6 +184,110 @@ notification service.
 ``value``
     The name of the breach, breach compilation or notification service.
 
+Images
+------
+
+``value``
+    The id that identifies the blob. This id is deterministic based on file
+    content.
+``filename``
+    This field is used if we have a well known filename for the content.
+``mime``
+    The image mimetype, like ``image/png`` or ``image/jpeg``.
+``width``
+    The width of the image.
+``height``
+    The height of the image.
+``created``
+    The date and time this image has been taken.
+``latitude``
+    Latitude this picture has been taken.
+``longitude``
+    Longitude this picture has been taken.
+``nudity``
+    A score that classifies nudity in this picture. The score goes from 0 to 2
+    and is commonly calculated with ``img_nudity``. A score above 1 means
+    nudity has been detected.
+``ahash``
+    The Mean (aHash) perceptual hash.
+``dhash``
+    The Gradient (dHash) perceptual hash.
+``phash``
+    The DCT (pHash) perceptual hash.
+
+Ports
+-----
+
+The status of a port on an ip address.
+
+``ip_addr_id``
+    The numeric id of an ipaddr struct.
+``ip_addr``
+    The actual ipaddr.
+``port``
+    The port number.
+``status``
+    The status of the port, either ``open`` or ``closed``.
+``banner``
+    The service banner we discovered on this port.
+``service``
+    The service that is running on this port.
+``version``
+    The version of the service running on this port.
+
+Netblocks
+---------
+
+A netblock is a network address range that has been allocated to an individual,
+organization or company. Those are commonly found when running whois lookups on
+an ip address.
+
+Consider the following example: Running a whois lookup on ``140.82.118.4`` (one
+of the addresses currently in use by github) returns that this address belongs
+to the netrange ``140.82.112.0 - 140.82.127.255``, so the netblock in this case
+is ``140.82.112.0/20``.
+
+``family``
+    This is either ``4`` or ``6`` and populated automatically.
+``value``
+    This is the network range in CIDR notation.
+``asn``
+    The number of the autonomous system this network belongs to.
+``as_org``
+    The organization of the autonomous system this network belongs to.
+``description``
+    This field isn't strictly defined and meant to be used as a human
+    meaningful name if available.
+
+CryptoAddrs
+-----------
+
+A cryptoaddr is any cryptocurrency address and not tied to a specific currency.
+
+``value``
+    The address string. This looks like ``1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2``.
+``currency``
+    The identifier for a specific currency. This is usually the ticker symbols,
+    like ``xbt``, ``zec`` or ``xmr``.
+``denominator``
+    Balance is tracked internally using 64 bit integers (signed, for technical reasons). Balance is supposed to be the lowest unit, so in case of bitcoin you'd write ``100,000,000`` satoshi instead of ``1`` bitcoin. Since this value is inconvinient to work with we're using the denominator to display values. In case of bitcoin you'd set it to ``8``.
+``balance``
+    The current balance of the address, in the lowest possible unit. In case of bitcoin this would be satoshis.
+``received``
+    The total amount of currency received by this address.
+``first_seen``
+    The first time currency was sent to this address.
+``last_withdrawal``
+    The last time a transaction signed by this address was observed.
+``description``
+    A human readable note for this address.
+
+Activity
+--------
+
+Activity is different from all other structs, have a look at the `Activity
+Section <activity.html>`_.
+
 Relations
 ---------
 

docs/usage.rst

@@ -256,3 +256,85 @@ You can reverse this using the scope command::
 .. hint::
    All entities have this field, you can refer to it in queries using
    ``unscoped=1``.
+
+Autonoscope
+-----------
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+~~~~~~~
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+~~~
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+~~~~
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

examples/maxmind.rs

@@ -1,61 +1,27 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
 use sn0int::paths;
-use std::fs;
 use std::net::IpAddr;
+use std::path::Path;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
 pub enum Args {
-    #[structopt(name="dl")]
-    Download(Download),
     #[structopt(name="asn")]
     Asn(AsnArgs),
     #[structopt(name="geoip")]
     GeoIP(GeoIPArgs),
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Download {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-impl Download {
-    fn run(&self) -> Result<()> {
-        let path = paths::cache_dir()?.join(&self.target);
-        if self.extract_only {
-            let body = fs::read(&self.url)?;
-            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
-        } else {
-            GeoIP::download(path, &self.filter, &self.url)?;
-        }
-        Ok(())
-    }
-}
-
 #[derive(Debug, StructOpt)]
 pub struct AsnArgs {
     ip: IpAddr,
 }
 
 impl AsnArgs {
-    fn run(&self) -> Result<()> {
-        let asndb = AsnDB::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = AsnDB::cache_path(cache_dir)?;
+        let asndb = AsnDB::open(&path)?;
 
         let asn = asndb.lookup(self.ip)?;
         println!("{:#?}", asn);
@@ -70,8 +36,9 @@ pub struct GeoIPArgs {
 }
 
 impl GeoIPArgs {
-    fn run(&self) -> Result<()> {
-        let geoip = GeoIP::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = GeoIP::cache_path(cache_dir)?;
+        let geoip = GeoIP::open(&path)?;
 
         let lookup = geoip.lookup(self.ip)?;
         println!("{:#?}", lookup);
@@ -84,10 +51,10 @@ impl GeoIPArgs {
 fn run() -> Result<()> {
     let args = Args::from_args();
     debug!("{:?}", args);
+    let cache_dir = paths::cache_dir()?;
     match args {
-        Args::Download(args) => args.run(),
-        Args::Asn(args) => args.run(),
-        Args::GeoIP(args) => args.run(),
+        Args::Asn(args) => args.run(&cache_dir),
+        Args::GeoIP(args) => args.run(&cache_dir),
     }
 }
 

migrations/2019-03-08-060037_images/down.sql

@@ -0,0 +1 @@
+DROP TABLE images;

migrations/2019-03-08-060037_images/up.sql

@@ -0,0 +1,21 @@
+CREATE TABLE images (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+
+    filename VARCHAR,
+    mime VARCHAR,
+    width INT,
+    height INT,
+    created DATETIME,
+
+    latitude FLOAT,
+    longitude FLOAT,
+
+    nudity FLOAT,
+    ahash VARCHAR,
+    dhash VARCHAR,
+    phash VARCHAR,
+
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT image_unique UNIQUE (value)
+);

migrations/2019-03-24-195306_email-names/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _emails_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO _emails_new (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM emails;
+
+DROP TABLE emails;
+ALTER TABLE _emails_new RENAME TO emails;
+
+PRAGMA foreign_keys=on;

migrations/2019-03-24-195306_email-names/up.sql

@@ -0,0 +1,2 @@
+-- Your SQL goes here
+ALTER TABLE emails ADD COLUMN displayname VARCHAR;

migrations/2019-05-30-142142_ports/down.sql

@@ -0,0 +1 @@
+DROP TABLE ports;

migrations/2019-05-30-142142_ports/up.sql

@@ -0,0 +1,17 @@
+CREATE TABLE ports (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);

migrations/2019-07-04-233515_autonoscope/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE autonoscope;

migrations/2019-07-04-233515_autonoscope/up.sql

@@ -0,0 +1,8 @@
+-- Your SQL goes here
+CREATE TABLE autonoscope (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    object VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    scoped BOOLEAN NOT NULL,
+    CONSTRAINT autonoscope_unique UNIQUE (object, value)
+);

migrations/2019-07-27-152215_netblocks/down.sql

@@ -0,0 +1 @@
+DROP TABLE netblocks;

migrations/2019-07-27-152215_netblocks/up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE netblocks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    asn INTEGER,
+    as_org VARCHAR,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2019-08-11-073709_misc-fields/down.sql

@@ -0,0 +1,41 @@
+PRAGMA foreign_keys=off;
+
+-- accounts
+CREATE TABLE _accounts_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);
+
+INSERT INTO _accounts_new (id, value, service, username, displayname, email, url, last_seen, unscoped)
+    SELECT id, value, service, username, displayname, email, url, last_seen, unscoped
+    FROM accounts;
+
+DROP TABLE accounts;
+ALTER TABLE _accounts_new RENAME TO accounts;
+
+-- networks
+CREATE TABLE _networks_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+INSERT INTO _networks_new (id, value, unscoped, latitude, longitude)
+    SELECT id, value, unscoped, latitude, longitude
+    FROM networks;
+
+DROP TABLE networks;
+ALTER TABLE _networks_new RENAME TO networks;
+
+PRAGMA foreign_keys=on;

migrations/2019-08-11-073709_misc-fields/up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE accounts ADD COLUMN phonenumber VARCHAR;
+ALTER TABLE accounts ADD COLUMN profile_pic VARCHAR;
+ALTER TABLE accounts ADD COLUMN birthday VARCHAR;
+ALTER TABLE networks ADD COLUMN description VARCHAR;

migrations/2019-11-19-154056_cryptoaddr/down.sql

@@ -0,0 +1 @@
+DROP TABLE cryptoaddrs;

migrations/2019-11-19-154056_cryptoaddr/up.sql

@@ -0,0 +1,13 @@
+CREATE TABLE cryptoaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    currency VARCHAR,
+    denominator INTEGER,
+    balance BIGINT,
+    received BIGINT,
+    first_seen DATETIME,
+    last_withdrawal DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2020-01-09-024234_activity/down.sql

@@ -0,0 +1,31 @@
+DROP TABLE activity;
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-01-09-024234_activity/up.sql

@@ -0,0 +1,44 @@
+CREATE TABLE activity (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    topic VARCHAR NOT NULL,
+    time DATETIME NOT NULL,
+    uniq VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    radius INTEGER,
+    content VARCHAR NOT NULL
+);
+CREATE UNIQUE INDEX activity_uniq ON activity(topic, uniq);
+CREATE INDEX activity_topic ON activity(topic);
+CREATE INDEX activity_time ON activity(time);
+CREATE INDEX activity_topic_time ON activity(topic, time);
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

modules/dev/arp-scan.lua

@@ -1,45 +0,0 @@
--- Description: Parse arp-scan output
--- Version: 0.3.0
--- License: GPL-3.0
-
--- sudo arp-scan -qglI wlp3s0
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        m = regex_find('(.+)\t(.+)', x)
-        if m ~= nil then
-            ipaddr = m[2]
-            mac = m[3]
-            now = datetime()
-
-            device_id = db_add('device', {
-                value=mac,
-                last_seen=now,
-            })
-            if last_err() then return end
-
-            db_add_ttl('network-device', {
-                network_id=network_id,
-                device_id=device_id,
-                ipaddr=ipaddr,
-                last_seen=now,
-            }, 300)
-            if last_err() then return end
-        end
-    end
-end

modules/dev/asn.lua

@@ -1,16 +0,0 @@
--- Description: Run a asn lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = asn_lookup(arg['value'])
-    if last_err() then return end
-
-    if arg['asn'] ~= lookup['asn'] or arg['as_org'] ~= lookup['as_org'] then
-        db_update('ipaddr', arg, {
-            asn=lookup['asn'],
-            as_org=lookup['as_org'],
-        })
-    end
-end

modules/dev/axfr.lua

@@ -1,159 +0,0 @@
--- Description: Try a zone transfer for subdomains
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function strip_root_dot(name)
-    m = regex_find("(.+)\\.$", name)
-    if last_err() then return end
-
-    if m == nil then
-        return name
-    else
-        return m[2]
-    end
-end
-
-function add_pointer(name)
-    local domain, domain_id, subdomain_id
-
-    -- select psl+1
-    domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-end
-
-function iter_axfr(zone, arg)
-    local name, r, m, domain
-
-    debug(arg)
-
-    name = arg[1]
-    r = arg[2]
-
-    -- select psl+1
-    domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-
-    -- this is a A record
-    if r['A'] ~= nil then
-        -- add the name and ip
-        ipaddr_id = db_add('ipaddr', {
-            family='4',
-            value=r['A'],
-        })
-        if last_err() then return end
-
-        db_add('subdomain-ipaddr', {
-            subdomain_id=subdomain_id,
-            ip_addr_id=ipaddr_id,
-        })
-        if last_err() then return end
-    end
-
-    if r['CNAME'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['CNAME'])
-        add_pointer(name)
-    end
-
-    if r['NS'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['NS'])
-        add_pointer(name)
-    end
-
-    if r['MX'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['MX'][2])
-        add_pointer(name:lower())
-    end
-end
-
-function iter_a(zone, arg)
-    local i, records, r
-
-    if arg == nil then return end
-
-    debug('nameserver: ' .. arg)
-    records = dns(zone, {
-        record='AXFR',
-        nameserver=arg .. ':53',
-        tcp=true,
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    i = 1
-    while records[i] ~= nil do
-        iter_axfr(zone, records[i])
-        if last_err() then return end
-        i = i+1
-    end
-end
-
-function iter_ns(zone, arg)
-    local i, records, r
-
-    if arg == nil then return end
-
-    records = dns(arg, {
-        record='A',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    i = 1
-    while records[i] ~= nil do
-        r = records[i][2]
-        iter_a(zone, r['A'])
-        if last_err() then return end
-        i = i+1
-    end
-end
-
-function run(arg)
-    records = dns(arg['value'], {
-        record='NS',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    i = 1
-    while records[i] ~= nil do
-        r = records[i][2]
-        iter_ns(arg['value'], r['NS'])
-        if last_err() then return end
-        i = i+1
-    end
-end

modules/dev/cname-harvest.lua

@@ -1,50 +0,0 @@
--- Description: Query for CNAMES to find subdomains
--- Version: 0.2.0
--- Source: subdomains
--- License: GPL-3.0
-
-function iter(r)
-    if r == nil then
-        return
-    end
-
-    m = regex_find("(.+)\\.$", r)
-    if last_err() then return end
-
-    if m == nil then
-        return
-    end
-    r = m[2]
-
-    domain = psl_domain_from_dns_name(r)
-    if last_err() then return end
-
-    domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=r,
-        })
-        if last_err() then return end
-    end
-end
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    i = 1
-    while records[i] ~= nil do
-        r = records[i][2]
-        iter(r['CNAME'])
-        if last_err() then return end
-        i = i+1
-    end
-end

modules/dev/ctlogs.lua

@@ -1,106 +0,0 @@
--- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.4.0
--- Source: domains
--- License: GPL-3.0
-
-function each_name(name)
-    local domain_id, psl_domain
-
-    if seen[name] == 1 then
-        return
-    end
-    seen[name] = 1
-    debug(name)
-
-    if name:find('*.') == 1 then
-        -- ignore wildcard domains
-        return
-    end
-
-    -- the cert might be valid for subdomains that do not belong to the
-    -- domain we started with
-    psl_domain = psl_domain_from_dns_name(name)
-    domain_id = domains[psl_domain]
-    if domain_id == nil then
-        if any_domain then
-            -- unknown domains should be added to database
-            domain_id = db_add('domain', {
-                value=psl_domain,
-            })
-        else
-            -- only use domains that are already in scope
-            domain_id = db_select('domain', psl_domain)
-        end
-
-        -- if we didn't get a valid id, skip
-        if domain_id == nil then
-            return
-        end
-
-        domains[psl_domain] = domain_id
-    end
-
-    db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-end
-
-function run(arg)
-    full = getopt('full') ~= nil
-    any_domain = getopt('any-domain') ~= nil
-
-    domains = {}
-    domains[arg['value']] = arg['id']
-
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://crt.sh/', {
-        query={
-            q='%.' .. arg['value'],
-            output='json'
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    certs = json_decode(resp['text'])
-    if last_err() then return end
-
-    seen = {}
-
-    i = 1
-    while i <= #certs do
-        c = certs[i]
-        debug(c)
-
-        if full then
-            -- fetch certificate
-            id = c['min_cert_id']
-            req = http_request(session, 'GET', 'https://crt.sh/', {
-                query={
-                    d=id .. '', -- TODO: find nicer way for tostring
-                }
-            })
-            resp = http_send(req)
-            if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-            -- iterate over all valid names
-            crt = x509_parse_pem(resp['text'])
-            if last_err() then return end
-            names = crt['valid_names']
-
-            j = 1
-            while j <= #names do
-                each_name(names[j])
-                j = j+1
-            end
-        else
-            each_name(c['name_value'])
-        end
-
-        i = i+1
-    end
-end

modules/dev/ddwrt-dhcp.lua

@@ -1,91 +0,0 @@
--- Description: Export dhcp leases from ddwrt webinterface
--- Version: 0.1.0
--- License: GPL-3.0
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    skip_redacted = not getopt('use-redacted')
-
-    router = getopt('router') -- http://192.0.2.1/
-    if not router then
-        return 'router option is missing (http://192.0.2.1/)'
-    end
-    username = getopt('user')
-    password = getopt('password')
-
-    options = {}
-    if username and password then
-        options['basic_auth'] = {username, password}
-    end
-
-    -- request status page
-    session = http_mksession()
-    url = url_join(router, '/Info.live.htm')
-    req = http_request(session, 'GET', url, options)
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then
-        return 'http error: ' .. resp['status']
-    end
-
-    txt = resp['text']
-    debug(txt)
-
-    -- extract leases from response
-    dhcp_section = regex_find('\\{dhcp_leases:: ([^\\}]+)\\}', txt)
-    if last_err() then return end
-    if not dhcp_section then
-        return 'Failed to get dhcp lease section'
-    end
-
-    leases = regex_find_all('\'([^\']+)\',\'([^\']+)\',\'([^\']+)\',\'[^\']+\',\'[^\']+\'', dhcp_section[2])
-    if last_err() then return end
-
-    now = datetime()
-
-    -- add devices to database
-    local i = 1
-    while i <= #leases do
-        local hostname = leases[i][2]
-        local ipaddr = leases[i][3]
-        local macaddr = leases[i][4]
-
-        debug({
-            hostname=hostname,
-            ipaddr=ipaddr,
-            macaddr=macaddr,
-        })
-
-        if skip_redacted and macaddr:match('^xx:xx:') then
-            info('Skipping redacted macaddr')
-        else
-            local device = {
-                value=macaddr,
-                last_seen=now,
-            }
-            if hostname ~= '*' then
-                device['hostname'] = hostname
-            end
-
-            local device_id = db_add('device', device)
-
-            db_add_ttl('network-device', {
-                network_id=network_id,
-                device_id=device_id,
-                ipaddr=ipaddr,
-                last_seen=now,
-            }, 120)
-        end
-
-        i = i+1
-    end
-end

modules/dev/dns-ptr.lua

@@ -1,32 +0,0 @@
--- Description: Run reverse dns lookups
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    if arg['family'] == '4' then
-        m = regex_find('^(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)$', arg['value'])
-
-        q = m[5] .. '.' .. m[4] .. '.' .. m[3] .. '.' .. m[2] .. '.in-addr.arpa'
-        debug('Resolving: ' .. q)
-
-        records = dns(q, {
-            record='PTR',
-        })
-        if last_err() then return end
-        if records['error'] ~= nil then return end
-        records = records['answers']
-
-        i = 1
-        while records[i] ~= nil do
-            r = records[i][2]
-            if r['PTR'] then
-                db_update('ipaddr', arg, {
-                    reverse_dns=r['PTR'],
-                })
-                if last_err() then return end
-            end
-            i = i+1
-        end
-    end
-end

modules/dev/dns-resolve.lua

@@ -1,43 +0,0 @@
--- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.2.0
--- Source: subdomains
--- License: GPL-3.0
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    -- update subdomain
-    resolvable = records['error'] == nil
-    if arg['resolvable'] ~= resolvable then
-        -- TODO: pass arg to function as well
-        db_update('subdomain', arg, {
-            resolvable=resolvable
-        })
-    end
-
-    if not resolvable then
-        return
-    end
-
-    records = records['answers']
-
-    i = 1
-    while records[i] ~= nil do
-        r = records[i][2]
-        if r['A'] ~= nil then
-            ipaddr_id = db_add('ipaddr', {
-                family='4',
-                value=r['A'],
-            })
-            if last_err() then return end
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=arg['id'],
-                ip_addr_id=ipaddr_id,
-            })
-            if last_err() then return end
-        end
-        i = i+1
-    end
-end

modules/dev/geoip.lua

@@ -1,10 +0,0 @@
--- Description: Run a geoip lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = geoip_lookup(arg['value'])
-    if last_err() then return end
-    db_update('ipaddr', arg, lookup)
-end

modules/dev/git-webroot.lua

@@ -1,28 +0,0 @@
--- Description: Search for git checkouts in webroot
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    url = url_join(arg['value'], '.git/HEAD')
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    reply = http_send(req)
-    if last_err() then return end
-
-    if reply['status'] ~= 200 then
-        return
-    end
-
-    if not regex_find('^ref: ', reply['text']) then
-        return
-    end
-
-    db_add('url', {
-        subdomain_id=arg['subdomain_id'],
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-    })
-end

modules/dev/github.lua

@@ -1,76 +0,0 @@
--- Description: Collect data from github profiles
--- Version: 0.1.0
--- Source: accounts:github.com
--- License: GPL-3.0
-
-function api_get(url)
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'invalid status code' end
-
-    local data = json_decode(resp['text'])
-    if last_err() then return end
-    return data
-end
-
-function scan4email(username)
-    local url = 'https://api.github.com/users/' .. username .. '/repos'
-    local repos = api_get(url)
-    if last_err() then return end
-
-    i = 1
-    while i <= #repos do
-        local repo = repos[i]
-        debug(repo)
-        local commits = api_get(repo['url'] .. '/commits')
-        if last_err() then return end
-
-        j = 1
-        while j <= #commits do
-            local commit = commits[j]
-            debug(commit)
-
-            if commit['author']['login'] == username then
-                -- name = commit['commit']['author']['name']
-                local email = commit['commit']['author']['email']
-                return email
-            end
-
-            if commit['committer']['login'] == username then
-                -- name = commit['commit']['committer']['name']
-                local email = commit['commit']['committer']['email']
-                return email
-            end
-
-            j = j+1
-        end
-
-        i = i+1
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    local url = 'https://api.github.com/users/' .. arg['username']
-
-    local data = api_get(url)
-    if last_err() then return end
-    debug(data)
-
-    -- company = data['company']
-    -- location = data['location']
-    -- homepage = data['blog']
-
-    local email = data['email']
-    if not email and not arg['email'] then
-        email = scan4email(arg['username'])
-        if last_err() then return end
-    end
-
-    db_update('account', arg, {
-        url=data['html_url'],
-        displayname=data['name'],
-        email=email,
-    })
-end

modules/dev/hackertarget-subdomains.lua

@@ -1,30 +0,0 @@
--- Description: Query hackertarget for subdomains of a domain
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.hackertarget.com/hostsearch/', {
-        query={
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    m = regex_find_all("([^,]+),.+\\n?", resp['text'])
-
-    i = 1
-    while i <= #m do
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=m[i][2]
-        })
-
-        i = i+1
-    end
-end

modules/dev/instagram.lua

@@ -1,36 +0,0 @@
--- Description: Collect data from instagram profiles
--- Version: 0.1.0
--- Source: accounts:instagram.com
--- License: GPL-3.0
-
-function run(arg)
-    local session = http_mksession()
-    local url = 'https://www.instagram.com/' .. arg['username'] .. '/'
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'invalid status code' end
-    local html = resp['text']
-
-    local ld = html_select(html, 'script[type="application/ld+json"]')
-    if last_err() then return end
-    debug(ld)
-
-    local ld = json_decode(ld['text'])
-    if last_err() then return end
-    debug(ld)
-
-    if ld['email'] then
-        db_add('email', {
-            value=ld['email'],
-        })
-    end
-
-    -- homepage=ld['url']
-
-    db_update('account', arg, {
-        displayname=ld['name'],
-        email=ld['email'],
-        url=url,
-    })
-end

modules/dev/isc-dhcpd-leases.lua

@@ -1,78 +0,0 @@
--- Description: Parse isc-dhcpd dhcpd.leases(5)
--- Version: 0.2.0
--- License: GPL-3.0
-
--- cat /var/lib/dhcpd/dhcpd.leases
-
-function add(lease)
-    if not lease['active'] then return end
-
-    now = datetime()
-
-    device_id = db_add('device', {
-        value=lease['mac'],
-        hostname=lease['hostname'],
-        last_seen=now,
-    })
-    if last_err() then return end
-
-    db_add_ttl('network-device', {
-        network_id=network_id,
-        device_id=device_id,
-        ipaddr=lease['ipaddr'],
-        last_seen=now,
-    }, 180)
-    if last_err() then return end
-end
-
-function each_line(x)
-    debug(x)
-    m = regex_find('^lease (\\S+) \\{\n$', x)
-    if m then
-        lease = {}
-        debug('ipaddr=' .. m[2])
-        lease['ipaddr'] = m[2]
-    end
-    m = regex_find('^\\s*hardware ethernet (\\S+);\n$', x)
-    if m then
-        debug('mac=' .. m[2])
-        lease['mac'] = m[2]
-    end
-    m = regex_find('^\\s*client-hostname \"(.+)\";\n$', x)
-    if m then
-        debug('hostname=' .. m[2])
-        lease['hostname'] = m[2]
-    end
-    m = regex_find('^\\s*binding state active;\n$', x)
-    if m then
-        debug('active=true')
-        lease['active'] = true
-    end
-    m = regex_find('^\\}\n$', x)
-    if m then
-        add(lease)
-    end
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        if not regex_find('^\\s*(#.*|\\s*)\n$', x) then
-            each_line(x)
-        end
-    end
-end

modules/dev/iw-station-dump.lua

@@ -1,77 +0,0 @@
--- Description: Parse iw station dump
--- Version: 0.2.0
--- License: GPL-3.0
-
--- iw dev wlan0 station dump
-
-function add(client)
-    if
-        client['authenticated'] == 'yes' and
-        client['authorized'] == 'yes' and
-        client['mac']
-    then
-        debug(client)
-
-        now = datetime()
-
-        device_id = db_add('device', {
-            value=client['mac'],
-            last_seen=now,
-        })
-        if last_err() then return end
-
-        db_add_ttl('network-device', {
-            network_id=network_id,
-            device_id=device_id,
-            last_seen=now,
-        }, 180)
-        if last_err() then return end
-    end
-
-    client = nil
-end
-
-function each_line(x)
-    debug(x)
-    m = regex_find('^Station (\\S+)', x)
-    if m then
-        if client then
-            add(client)
-        end
-        client = {}
-        client['mac'] = m[2]
-        debug('mac=' .. m[2])
-    end
-
-    m = regex_find('^\\s+([^:]+):\\s*(.+)\n$', x)
-    if m and client then
-        client[m[2]] = m[3]
-        debug(m[2] .. '=' .. m[3])
-    end
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    client = nil
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        each_line(x)
-    end
-
-    if client then
-        add(client)
-    end
-end

modules/dev/namechk.lua

@@ -1,90 +0,0 @@
--- Description: Find accounts by username with namechk.com
--- Version: 0.1.0
--- Source: accounts
--- License: GPL-3.0
-
-function get_services(html)
-    local divs = html_select_list(html, '.service')
-    if last_err() then return end
-
-    local services = {}
-
-    local i = 1
-    while i <= #divs do
-        services[i] = divs[i]['attrs']['data-name']
-        i = i+1
-    end
-
-    return services
-end
-
-function run(arg)
-    -- setup session
-    local session = http_mksession()
-    local req = http_request(session, 'GET', 'https://namechk.com/', {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    local token = html_select(resp['text'], 'input[name="authenticity_token"]')
-    local auth_token = token['attrs']['value']
-
-        local headers = {}
-        headers['X-CSRF-Token'] = authenticity_token
-
-    local services = get_services(resp['text'])
-    debug({
-        auth_token=auth_token,
-        services=services,
-    })
-
-    -- trigger the scan
-    local req = http_request(session, 'POST', 'https://namechk.com/', {
-            headers=headers,
-        form={
-            authenticity_token=auth_token,
-            q=arg['username'],
-        }
-    })
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-    debug(resp)
-
-    local scan = json_decode(resp['text'])
-    if last_err() then return end
-    local scan_token = scan['valid']
-
-    -- get results
-    local i = 1
-    while i <= #services do
-        debug(services[i])
-
-        local req = http_request(session, 'POST', 'https://namechk.com/services/check', {
-            headers=headers,
-            form={
-                token=scan_token,
-                fat=auth_token,
-                service=services[i],
-            }
-        })
-        local resp = http_send(req)
-        if last_err() then return end
-
-        if resp['status'] == 200 then
-            local acc = json_decode(resp['text'])
-            if last_err() then return end
-            debug(acc)
-
-            if acc ~= nil and not acc['available'] and acc['status'] == 'unavailable' then
-                db_add('account', {
-                    service=services[i],
-                    username=arg['username'],
-                    url=acc['callback_url'],
-                })
-            end
-        end
-
-        i = i+1
-    end
-end

modules/dev/otx-subdomains.lua

@@ -1,32 +0,0 @@
--- Description: Query alienvault otx passive dns for subdomains of a domain
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    url = 'https://otx.alienvault.com/api/v1/indicators/domain/' .. arg['value'] .. '/passive_dns'
-
-    req = http_request(session, 'GET', url, {})
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['passive_dns']
-
-    i = 1
-    while o[i] do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x['hostname'],
-        })
-
-        i = i+1
-    end
-end

modules/dev/passive-arp.lua

@@ -1,79 +0,0 @@
--- Description: Passive arp-scanner with sniffglue
--- Version: 0.1.0
--- License: GPL-3.0
-
--- sudo sniffglue -jv enp0s25
-
-function each_frame(frame)
-    if not frame['Ether'] then return end
-
-    local arp = frame['Ether'][2]['Arp']
-    if not arp then return end
-
-    if arp['Request'] then
-        arp = arp['Request']
-    elseif arp['Reply'] then
-        arp = arp['Reply']
-    else
-        -- unknown, abort
-        return
-    end
-
-    debug(arp)
-
-    -- TODO: this might change to a string in the future
-    local mac = mac(arp['src_mac'])
-    local ipaddr = arp['src_addr']
-    debug({src_mac=mac, src_addr=ipaddr})
-
-    local now = datetime()
-
-    local device_id = db_add('device', {
-        value=mac,
-        last_seen=now,
-    })
-    if last_err() then return end
-
-    db_add_ttl('network-device', {
-        network_id=network_id,
-        device_id=device_id,
-        ipaddr=ipaddr,
-        last_seen=now,
-    }, 120)
-    if last_err() then return end
-end
-
-function mac(m)
-    return
-        hex({m[1]}) .. ':' ..
-        hex({m[2]}) .. ':' ..
-        hex({m[3]}) .. ':' ..
-        hex({m[4]}) .. ':' ..
-        hex({m[5]}) .. ':' ..
-        hex({m[6]})
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        local x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        local frame = json_decode(x)
-        if last_err() then return end
-
-        each_frame(frame)
-        if last_err() then return end
-    end
-end

modules/dev/passive-spider.lua

@@ -1,63 +0,0 @@
--- Description: Scrape known http responses for urls
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function entry(parent, href)
-    -- TODO: parse mailto:foo@example.com?subject=asdf
-    -- TODO: parse tel:+4912345
-    -- TODO: allow discovering 3rd-party domains
-    -- TODO: maybe record urls as well
-
-    local psl, parts, url, host
-
-    if href == nil then
-        return
-    end
-
-    url = url_join(parent, href)
-    if last_err() then return clear_err() end
-    if url:match('^https?://') == nil then
-        return
-    end
-
-    parts = url_parse(url)
-    if last_err() then return end
-    host = parts['host']
-    psl = psl_domain_from_dns_name(host)
-
-
-    domain_id = db_select('domain', psl)
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=host,
-        })
-    end
-end
-
-function run(arg)
-    if arg['body'] == nil or #arg['body'] == 0 then
-        return
-    end
-
-    body = utf8_decode(arg['body'])
-    if last_err() then return end
-
-    links = html_select_list(body, 'a')
-    if last_err() then return end
-
-    if #links == 0 then
-        return
-    end
-
-    -- process html links
-    i = 1
-    while i <= #links do
-        href = links[i]['attrs']['href']
-
-        entry(arg['value'], href)
-
-        i = i+1
-    end
-end

modules/dev/pgp-keyserver.lua

@@ -1,58 +0,0 @@
--- Description: Query pgp keyserver for email addresses
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    --lookup_url = 'https://pgp.mit.edu/pks/lookup'
-    lookup_url = 'https://sks-keyservers.net/pks/lookup'
-
-    req = http_request(session, 'GET', lookup_url, {
-        query={
-            search=arg['value'],
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    links = html_select_list(resp['text'], 'a')
-    i = 1
-    while i <= #links do
-        href = links[i]['attrs']['href']
-
-        if href:find('/pks/lookup%?op=get&search=') == 1 then
-            url = url_join(lookup_url, href)
-
-            req = http_request(session, 'GET', url, {})
-
-            resp = http_send(req)
-            -- TODO: do not abort script if one attempt fails
-            if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-            pubkey = pgp_pubkey_armored(resp['text'])
-
-            -- print(pubkey)
-
-            -- TODO: ensure at least one email matches our target domain
-            if pubkey['uids'] then
-                j = 1
-                while j <= #pubkey['uids'] do
-                    m = regex_find("<([^< ]+@[^< ]+)>$", pubkey['uids'][j])
-                    if m then
-                        db_add('email', {
-                            value=m[2],
-                        })
-                    end
-                    j = j+1
-                end
-            end
-        end
-
-        i = i+1
-    end
-end

modules/dev/phpmyadmin.lua

@@ -1,106 +0,0 @@
--- Description: Search for phpmyadmin
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    paths = {
-        "phpmyadmin/index.php",
-        "phpMyAdmin/index.php",
-        "pmd/index.php",
-        "pma/index.php",
-        "PMA/index.php",
-        "PMA2/index.php",
-        "pmamy/index.php",
-        "pmamy2/index.php",
-        "mysql/index.php",
-        "admin/index.php",
-        "db/index.php",
-        "dbadmin/index.php",
-        "web/phpMyAdmin/index.php",
-        "admin/pma/index.php",
-        "admin/PMA/index.php",
-        "admin/mysql/index.php",
-        "admin/mysql2/index.php",
-        "admin/phpmyadmin/index.php",
-        "admin/phpMyAdmin/index.php",
-        "admin/phpmyadmin2/index.php",
-        "mysqladmin/index.php",
-        "mysql-admin/index.php",
-        "mysql_admin/index.php",
-        "phpadmin/index.php",
-        "phpAdmin/index.php",
-        "phpmyadmin0/index.php",
-        "phpmyadmin1/index.php",
-        "phpmyadmin2/index.php",
-        "phpMyAdmin-4.4.0/index.php",
-        "myadmin/index.php",
-        "myadmin2/index.php",
-        "xampp/phpmyadmin/index.php",
-        "phpMyadmin_bak/index.php",
-        "www/phpMyAdmin/index.php",
-        "tools/phpMyAdmin/index.php",
-        "phpmyadmin-old/index.php",
-        "phpMyAdminold/index.php",
-        "phpMyAdmin.old/index.php",
-        "pma-old/index.php",
-        "claroline/phpMyAdmin/index.php",
-        "typo3/phpmyadmin/index.php",
-        "phpma/index.php",
-        "phpmyadmin/phpmyadmin/index.php",
-        "phpMyAdmin/phpMyAdmin/index.php",
-        "phpMyAbmin/index.php",
-        "phpMyAdmin__/index.php",
-        "phpMyAdmin+++---/index.php",
-        "v/index.php",
-        "phpmyadm1n/index.php",
-        "phpMyAdm1n/index.php",
-        "shaAdmin/index.php",
-        "phpMyadmi/index.php",
-        "phpMyAdmion/index.php",
-        "MyAdmin/index.php",
-        "phpMyAdmin1/index.php",
-        "phpMyAdmin123/index.php",
-        "pwd/index.php",
-        "phpMyAdmina/index.php",
-        "program/index.php",
-        "shopdb/index.php",
-        "phppma/index.php",
-        "phpmy/index.php",
-        "mysql/admin/index.php",
-        "mysql/dbadmin/index.php",
-        "mysql/sqlmanager/index.php",
-        "mysql/mysqlmanager/index.php",
-        "wp-content/plugins/portable-phpmyadmin/wp-pma-mod/index.php",
-    }
-
-    session = http_mksession()
-
-    i = 1
-    while i <= #paths do
-        p = paths[i]
-        url = url_join(arg['value'], p)
-        debug(url)
-
-        req = http_request(session, 'GET', url, {
-            timeout=5000
-        })
-        reply = http_send(req)
-        debug(reply)
-
-        if last_err() then
-            clear_err()
-        else
-            if reply['status'] == 200 then
-                db_add('url', {
-                    subdomain_id=arg['subdomain_id'],
-                    value=url,
-                    status=reply['status'],
-                    body=reply['text'],
-                })
-            end
-        end
-
-        i = i+1
-    end
-end

modules/dev/smtp-check.lua

@@ -1,90 +0,0 @@
--- Description: Verify email address by asking the smtp server
--- Version: 0.1.0
--- Source: emails
--- License: GPL-3.0
-
-function find_mx(domain)
-    local records, i, r
-
-    records = dns(domain, {
-        record='MX',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-    -- debug(records)
-
-    i = 1
-    while i <= #records do
-        r = records[i][2]['MX']
-        if r then
-            debug('mx: ' .. r[2])
-            return r[2]
-        end
-        i = i+1
-    end
-end
-
-function run(arg)
-    -- extract domain
-    domain = arg['value']:match('@(.*)')
-    if doman ~= nil then
-        -- malformed domain
-        return
-    end
-
-    -- mx lookup
-    mx = find_mx(domain)
-    if last_err() then return end
-    if not mx then return end
-
-    -- create connection
-    c = sock_connect(mx, 25, {})
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- send hello
-    sock_sendline(c, 'ehlo localhost')
-    if last_err() then return end
-
-    l = sock_recvline_regex(c, '^250 ')
-    if last_err() then return end
-    debug(l)
-
-    -- send email
-    sock_sendline(c, 'mail from:<root@localhost>')
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- send rcpt
-    sock_sendline(c, 'rcpt to:<' .. arg['value'] .. '>')
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- check status
-    verified = nil
-    if l:match('^2') then
-        debug('email is valid')
-        verified = true
-    elseif l:match('^5') then
-        debug('email is invalid')
-        verified = false
-    elseif l:match('^4') then
-        debug('unknown status, temporary delivery failure')
-    end
-
-    if verified ~= nil then
-        db_update('email', arg, {
-            valid=verified,
-        })
-    end
-end

modules/dev/threatminer-ipaddr.lua

@@ -1,52 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of an ip address
--- Version: 0.2.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- TODO: add option to filter old entries based on last_seen
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/host.php', {
-        query={
-            rt='2',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    i = 1
-    while o[i] do
-        x = o[i]
-
-        domain = psl_domain_from_dns_name(x['domain'])
-        -- TODO: if this fails, skip this entry instead
-        if last_err() then return end
-
-        domain_id = db_add('domain', {
-            value=domain,
-        })
-
-        if domain_id ~= nil then
-            subdomain_id = db_add('subdomain', {
-                domain_id=domain_id,
-                value=x['domain'],
-            })
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=subdomain_id,
-                ip_addr_id=arg['id'],
-            })
-        end
-
-        i = i+1
-    end
-end

modules/dev/threatminer-subdomains.lua

@@ -1,35 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of a domain
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/domain.php', {
-        query={
-            rt='5',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    i = 1
-    while o[i] do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x,
-        })
-
-        i = i+1
-    end
-end

modules/dev/thunderbird-autoconfig.lua

@@ -1,48 +0,0 @@
--- Description: Query thunderbird autoconfig db for subdomains
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- check if an autoconfig exists without disclosing our target yet
-    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/', {})
-    resp = http_send(req)
-    if last_err() then return end
-
-    if resp['status'] ~= 200 then
-        return 'index request failed'
-    end
-
-    if resp['text']:find(arg['value'], 1, true) == nil then
-        debug('no autoconfig available')
-        return
-    end
-
-    -- request config
-    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/' .. arg['value'], {})
-    resp = http_send(req)
-    if last_err() then return end
-
-    m = regex_find_all('<hostname>([^<]+)</hostname>', resp['text'])
-
-    i = 1
-    while i <= #m do
-        subdomain = m[i][2]
-
-        domain = psl_domain_from_dns_name(subdomain)
-        if last_err() then return end
-
-        domain_id = db_select('domain', domain)
-        if last_err() then return end
-
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=subdomain,
-        })
-        if last_err() then return end
-
-        i = i+1
-    end
-end

modules/dev/twilio-lookup.lua

@@ -1,48 +0,0 @@
--- Description: Retrieve additional information about a phone number
--- Version: 0.1.0
--- Source: phonenumbers
--- Keyring-Access: twilio
--- License: GPL-3.0
-
-function run(arg)
-    number = url_escape(arg['value'])
-    --url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number
-    url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number .. '?Type=carrier&Type=caller-name'
-
-    --debug(url)
-
-    key = keyring('twilio')[1]
-    if not key then
-        return 'Missing required twilio access key'
-    end
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {
-        basic_auth={key['access_key'], key['secret_key']},
-    })
-    reply = http_send(req)
-    if last_err() then return end
-
-    if reply['status'] ~= 200 then
-        return 'api returned error'
-    end
-
-    v = json_decode(reply['text'])
-    if last_err() then return end
-    debug(v)
-
-    update = {}
-    update['country'] = v['country_code']
-
-    if v['carrier'] then
-        update['carrier'] = v['carrier']['name']
-        update['line'] = v['carrier']['type']
-    end
-
-    if v['caller_name'] then
-        update['caller_name'] = v['caller_name']['caller_name']
-        update['caller_type'] = v['caller_name']['caller_type']
-    end
-
-    db_update('phonenumber', arg, update)
-end

modules/dev/url-scan.lua

@@ -1,45 +0,0 @@
--- Description: Scan subdomains for websites
--- Version: 0.3.0
--- Source: subdomains
--- License: GPL-3.0
-
-function request(subdomain_id, url)
-    req = http_request(session, 'GET', url, {
-        timeout=5000
-    })
-    reply = http_send(req)
-
-    if last_err() then
-        clear_err()
-        return
-    end
-
-    obj = {
-        subdomain_id=subdomain_id,
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-    }
-
-    redirect = reply['headers']['location']
-    if redirect then
-        obj['redirect'] = url_join(url, redirect)
-    end
-
-    db_add('url', obj)
-
-    -- debug(reply['status'])
-    -- debug(reply['headers']['location'])
-    -- debug(reply['text'])
-end
-
-function run(arg)
-    domain = arg['value']
-
-    session = http_mksession()
-
-    request(arg['id'], 'http://' .. domain .. '/')
-    if last_err() then return end
-    request(arg['id'], 'https://' .. domain .. '/')
-    if last_err() then return end
-end

modules/dev/waybackurls.lua

@@ -1,57 +0,0 @@
--- Description: Discover subdomains from wayback machine
--- Version: 0.3.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-
-    domain = arg['value']
-    url = 'https://web.archive.org/cdx/search/cdx?url=*.' .. domain .. '/*&output=json&collapse=urlkey'
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-
-    -- no known urls
-    if o[1] == nil then
-        return
-    end
-
-    -- ensure the api response is still what we expect
-    if o[1][3] == nil then
-        return 'api returned unexpected json format'
-    end
-
-    seen = {}
-
-    i = 2
-    while o[i] do
-        url = o[i][3]
-        debug(url)
-        parts = url_parse(url)
-
-        if last_err() then
-            clear_err()
-            error("Failed to parse url: " .. json_encode(url))
-        else
-            subdomain = parts['host']
-            subdomain, _ = subdomain:gsub('%.$', '')
-
-            if seen[subdomain] == nil then
-                db_add('subdomain', {
-                    domain_id=arg['id'],
-                    value=parts['host'],
-                })
-                if last_err() then return end
-                seen[subdomain] = 1
-            end
-        end
-
-        i = i+1
-    end
-end

modules/dev/well-known-uris.lua

@@ -1,64 +0,0 @@
--- Description: Scan for known /.well-known/ locations
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    -- https://www.iana.org/assignments/well-known-uris/well-known-uris.xhtml
-    -- https://en.wikipedia.org/wiki/List_of_/.well-known/_services_offered_by_webservers
-
-    -- TODO: check if every location causes a 200/redirect
-
-    locations = {
-        {path='security.txt'}, -- expect 200
-        {path='dnt-policy.txt'}, -- expect 200
-        {path='caldav', redirect=true}, -- expect redirect
-        {path='autoconfig/mail/config-v1.1.xml'}, -- expect 200
-        {path='assetlinks.json'}, -- expect 200
-        {path='apple-app-site-association'},  -- expect 200
-        {path='keybase.txt'}, -- expect 200
-        {path='apple-developer-merchantid-domain-association'}, -- expect 200
-        {path='openpgpkey'}, -- expect 200
-        {path='change-password', redirect=true}, -- expect redirect
-    }
-
-    session = http_mksession()
-
-    i = 1
-    while i <= #locations do
-        path = locations[i]['path']
-        expect_redirect = locations[i]['redirect']
-
-        url = url_join(arg['value'], '/.well-known/' .. path)
-        debug(url)
-
-        req = http_request(session, 'GET', url, {
-            timeout=5000,
-        })
-        reply = http_send(req)
-        debug(reply)
-
-        if last_err() then
-            clear_err()
-        else
-            status = reply['status']
-            if (status == 200 and not expect_redirect) or (expect_redirect and status >= 300 and status < 400) then
-                obj = {
-                    subdomain_id=arg['subdomain_id'],
-                    value=url,
-                    status=reply['status'],
-                    body=reply['text'],
-                }
-
-                redirect = reply['headers']['location']
-                if redirect then
-                    obj['redirect'] = url_join(url, redirect)
-                end
-
-                db_add('url', obj)
-            end
-        end
-
-        i = i+1
-    end
-end

modules/harness/activity-ping.lua

@@ -0,0 +1,29 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    while true do
+        db_activity({
+            topic=topic,
+            time=sn0int_time(),
+            uniq=uniq,
+            latitude=lat,
+            longitude=lon,
+            radius=radius,
+            content={
+                msg='ohai',
+            },
+        })
+        sleep(5)
+    end
+end

modules/harness/add-all.lua

@@ -0,0 +1,107 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('adding domain')
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+    if last_err() then return end
+
+    info('adding subdomain')
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value='example.com',
+    })
+    if last_err() then return end
+
+    info('adding ipaddr')
+    ipaddr_id = db_add('ipaddr', {
+        value='192.0.2.1',
+    })
+    if last_err() then return end
+
+    info('adding device')
+    device_id = db_add('device', {
+        value='ff:ff:ff:ff:ff:ff',
+    })
+    if last_err() then return end
+
+    info('adding network')
+    network_id = db_add('network', {
+        value='myssid',
+    })
+    if last_err() then return end
+
+    info('adding email')
+    email_id = db_add('email', {
+        value='foo@example.com',
+    })
+    if last_err() then return end
+
+    info('adding phonenumber')
+    phonenumber_id = db_add('phonenumber', {
+        value='+4912345678',
+    })
+    if last_err() then return end
+
+    info('adding breach')
+    breach_id = db_add('breach', {
+        value='hack the planet',
+    })
+    if last_err() then return end
+
+    info('adding account')
+    account_id = db_add('account', {
+        service='github.com',
+        username='kpcyrd',
+    })
+    if last_err() then return end
+
+    info('adding image')
+    blob = create_blob('abc')
+    image_id = db_add('image', {
+        value=blob,
+    })
+    if last_err() then return end
+
+    info('adding port')
+    port_id = db_add('port', {
+        ip_addr_id=ipaddr_id,
+        ip_addr='192.0.2.1',
+        port=443,
+        protocol='tcp',
+        status='open',
+    })
+    if last_err() then return end
+
+    info('adding url')
+    url_id = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://www.example.com/a/b',
+        body='<html></html>',
+    })
+    if last_err() then return end
+
+    info('adding breach_email')
+    db_add('breach-email', {
+        breach_id=breach_id,
+        email_id=email_id,
+    })
+    if last_err() then return end
+
+    info('adding network_device')
+    db_add('network-device', {
+        network_id=network_id,
+        device_id=device_id,
+    })
+    if last_err() then return end
+
+    info('adding subdomain_ipaddr')
+    db_add('subdomain-ipaddr', {
+        subdomain_id=subdomain_id,
+        ip_addr_id=ipaddr_id,
+    })
+    if last_err() then return end
+end

modules/harness/geo-polygon-contains.lua

@@ -0,0 +1,74 @@
+-- Description: demonstrate geofencing with polygons
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    hamburg = {
+        { lat=53.63975308945899,  lon=9.764785766601562  },
+        { lat=53.59494998253459,  lon=9.827270507812     },
+        { lat=53.663153974456456, lon=9.9151611328125    },
+        { lat=53.65582987649682,  lon=9.976272583007812  },
+        { lat=53.68613523817129,  lon=9.992752075195312  },
+        { lat=53.68674518938816,  lon=10.051460266113281 },
+        { lat=53.72495117617815,  lon=10.075492858886719 },
+        { lat=53.71946627930625,  lon=10.118408203125    },
+        { lat=53.743635083157756, lon=10.164413452148438 },
+        { lat=53.73104466704585,  lon=10.202865600585938 },
+        { lat=53.676781546441546, lon=10.16304016113281  },
+        { lat=53.632832079199474, lon=10.235824584960938 },
+        { lat=53.608803292930894, lon=10.2008056640625   },
+        { lat=53.578646152866504, lon=10.208358764648438 },
+        { lat=53.57212285981298,  lon=10.163726806640625 },
+        { lat=53.52071674896369,  lon=10.18707275390625  },
+        { lat=53.52643162253097,  lon=10.224151611328125 },
+        { lat=53.44062753992289,  lon=10.347747802734375 },
+        { lat=53.38824275010831,  lon=10.248870849609375 },
+        { lat=53.38824275010831,  lon=10.15960693359375  },
+        { lat=53.44635321212876,  lon=10.064849853515625 },
+        { lat=53.40595029739904,  lon=9.985198974609375  },
+        { lat=53.42385506057106,  lon=9.951210021972656  },
+        { lat=53.41843327091211,  lon=9.944171905517578  },
+        { lat=53.41812635648326,  lon=9.927349090576172  },
+        { lat=53.412294561442884, lon=9.917736053466797  },
+        { lat=53.41464783813818,  lon=9.901256561279297  },
+        { lat=53.443490472483326, lon=9.912586212158201  },
+        { lat=53.45177144115704,  lon=9.897651672363281  },
+        { lat=53.43633277935392,  lon=9.866924285888672  },
+        { lat=53.427639673754776, lon=9.866409301757812  },
+        { lat=53.427639673754776, lon=9.858856201171875  },
+        { lat=53.46710230573499,  lon=9.795513153076172  },
+        { lat=53.49039461941655,  lon=9.795341491699219  },
+        { lat=53.49029248806277,  lon=9.77903366088867   },
+        { lat=53.49856433088649,  lon=9.780235290527344  },
+        { lat=53.5078554643033,   lon=9.758434295654297  },
+        { lat=53.545407634092975, lon=9.759807586669922  },
+        { lat=53.568147234570084, lon=9.633293151855469  },
+        { lat=53.58802162343514,  lon=9.655780792236328  },
+        { lat=53.568351121879815, lon=9.727706909179688  },
+        { lat=53.60921067445695,  lon=9.737663269042969  },
+    }
+
+    points = {
+        {
+            name='Alice',
+            lat=52.52437,
+            lon=13.41053,
+        }, {
+            name='Bob',
+            lat=53.551085,
+            lon=9.993682,
+        }, {
+            name='Charlie',
+            lat=40.726662,
+            lon=-74.036677,
+        }
+    }
+
+    for i=1, #points do
+        if geo_polygon_contains(hamburg, points[i]) then
+            info('[INSIDE ] ' .. points[i]['name'])
+        else
+            info('[OUTSIDE] ' .. points[i]['name'])
+        end
+    end
+end

modules/harness/google-tls.lua

@@ -0,0 +1,29 @@
+-- Description: Test various tls functions
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('sending https request to google.com')
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://google.com/', {})
+    r = http_send(req)
+    if last_err() then return end
+    debug(r)
+
+    info('creating tls socket to google.com')
+    sock = sock_connect('google.com', 443, {
+        tls=true,
+    })
+    if last_err() then return end
+    debug(sock)
+
+    info('creating socket to google.com, wrapping afterwards')
+    sock = sock_connect('google.com', 443, {})
+    if last_err() then return end
+    tls = sock_upgrade_tls(sock, {
+        sni_value='google.com',
+    })
+    if last_err() then return end
+    debug(sock)
+    debug(tls)
+end

modules/harness/import-subdomains.lua

@@ -0,0 +1,29 @@
+-- Description: Import subdomains from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lu
+
+    while true do
+        local line = stdin_readline()
+        if line == nil then
+            break
+        end
+        -- strip newline
+        local m = regex_find('.+', line)
+        if m then
+            local subdomain = m[1]
+
+            local domain = psl_domain_from_dns_name(subdomain)
+            local domain_id = db_add('domain', {
+                value=domain,
+            })
+
+            db_add('subdomain', {
+                domain_id=domain_id,
+                value=subdomain,
+            })
+        end
+    end
+end

modules/harness/onion-http.lua

@@ -0,0 +1,16 @@
+-- Description: Send a request to a hidden service
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local session = http_mksession()
+    local req = http_request(session, 'GET', 'http://expyuzz4wqqyqhjn.onion/', {
+        proxy='127.0.0.1:9050',
+    })
+    local r = http_fetch(req)
+    if last_err() then return end
+
+    local title = html_select(r['text'], 'title')
+    if last_err() then return end
+    info(title['html'])
+end

modules/harness/pgp-stdin.lua

@@ -0,0 +1,9 @@
+-- Description: Decode armored pgp key from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local pubkey = stdin_read_to_end()
+    pubkey = pgp_pubkey_armored(pubkey)
+    info(pubkey)
+end

modules/harness/port.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    ip_addr = '192.168.1.2'
+    ip_addr_id = db_add('ipaddr', {
+        value=ip_addr,
+    })
+    db_add('port', {
+        ip_addr_id=ip_addr_id,
+        ip_addr=ip_addr,
+        protocol='tcp',
+        port=4444,
+        status='open',
+    })
+end

modules/harness/ratelimit.lua

@@ -0,0 +1,11 @@
+-- Description: Run script with a global ratelimit
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- this shouldn't complete in less than 5 seconds
+    for i=1, 20 do
+        ratelimit_throttle('foo', 4, 1000)
+        info(sn0int_time())
+    end
+end

modules/harness/socket-ping.lua

@@ -0,0 +1,33 @@
+-- Description: Connect somewhere and send a ping every 3s
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+INTERVAL = 3
+
+function run()
+    local sock = sock_connect('127.0.0.1', 4444, {
+        read_timeout=INTERVAL,
+    })
+    if last_err() then return end
+
+    local last_ping = time_unix()
+    while true do
+        local now = time_unix()
+        local sleep = last_ping + INTERVAL - now
+
+        if sleep <= 0 then
+            sock_send(sock, sn0int_time() .. ' ping\n')
+            last_ping = now
+            sleep = INTERVAL
+        end
+
+        sock_options(sock, {
+            read_timeout=sleep,
+        })
+        if last_err() then return end
+
+        local buf = sock_recv(sock)
+        if last_err() then return end
+        info(buf)
+    end
+end

modules/harness/tls-insecure.lua

@@ -0,0 +1,16 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    sock = sock_connect('expired.badssl.com', 443, {})
+    if last_err() then return end
+
+    tls = sock_upgrade_tls(sock, {
+        sni_value='expired.badssl.com',
+        disable_tls_verify=true,
+    })
+    if last_err() then return end
+
+    info(tls)
+end

modules/harness/tls.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    sock = sock_connect('badssl.com', 443, {})
+    if last_err() then return end
+
+    tls = sock_upgrade_tls(sock, {
+        sni_value='badssl.com',
+    })
+    if last_err() then return end
+
+    info(tls)
+
+    info(x509_parse_pem(tls['cert']))
+end

modules/harness/tux-img.lua

@@ -0,0 +1,18 @@
+-- Description: Download an image
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+
+    req = http_request(session, 'GET', 'https://www.kernel.org/theme/images/logos/tux.png', {
+        into_blob=true,
+    })
+    r = http_fetch(req)
+    if last_err() then return end
+
+    debug(r)
+    db_add('image', {
+        value=r['blob'],
+    })
+end

modules/harness/warn.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run()
+    warn('testing')
+    warn_once('testing once')
+    warn_once('testing once')
+end

modules/harness/ws-ping.lua

@@ -0,0 +1,33 @@
+-- Description: Connect somewhere and send a ping every 3s
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+INTERVAL = 3
+
+function run()
+    local sock = ws_connect('ws://127.0.0.1:8080', {
+        read_timeout=INTERVAL,
+    })
+    if last_err() then return end
+
+    local last_ping = time_unix()
+    while true do
+        local now = time_unix()
+        local sleep = last_ping + INTERVAL - now
+
+        if sleep <= 0 then
+            ws_send_text(sock, sn0int_time() .. ' ping\n')
+            last_ping = now
+            sleep = INTERVAL
+        end
+
+        ws_options(sock, {
+            read_timeout=sleep,
+        })
+        if last_err() then return end
+
+        local buf = ws_recv_text(sock)
+        if last_err() then return end
+        info(buf)
+    end
+end

modules/harness/ws.lua

@@ -0,0 +1,23 @@
+-- Description: Create an echo websocket connection
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local target = 'ws://echo.websocket.org'
+
+    info('connecting to ' .. target)
+    local sock = ws_connect(target, {})
+    if last_err() then return end
+
+    info('sending')
+    ws_send_text(sock, 'ohai wurld')
+    if last_err() then return end
+
+    info('recieving')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'ohai wurld' then
+        return 'echo failed, got: ' .. msg
+    end
+end

modules/harness/wss.lua

@@ -0,0 +1,30 @@
+-- Description: Create an encrypted websocket connection
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- local target = 'ws://echo.websocket.org' -- doesn't support proper ciphers
+    local target = 'wss://rocket.events.ccc.de/sockjs/258/whi0yr1y/websocket'
+
+    info('connecting to ' .. target)
+    local sock = ws_connect(target, {})
+    if last_err() then return end
+
+    info('recieving 1/2')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'o' then
+        return 'recieve failed, got ' .. msg
+    end
+
+    info('recieving 2/2')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'a["{\\"server_id\\":\\"0\\"}"]' then
+        return 'recieve failed, got ' .. msg
+    end
+
+    info('handshake succeeded')
+end

modules/harness/xss.lua

@@ -0,0 +1,8 @@
+-- Description: jaVasCript:/*-/*`/*\`/*'/*"/**/(/* */oNcliCk=alert() )//%0D%0A%0d%0a//</stYle/</titLe/</teXtarEa/</scRipt/--!>\x3csVg/<sVg/oNloAd=alert()//>\x3e
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run()
+    -- TODO your code here
+end

sn0int-common/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "sn0int-common"
-version = "0.5.0"
-description = "Common code for sn0int"
+version = "0.10.0"
+description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
@@ -10,7 +10,7 @@ edition = "2018"
 [dependencies]
 serde = "1.0"
 serde_derive = "1.0"
-#rocket_failure = { path = "../../../rocket_failure" }
-rocket_failure = "0.1.1"
+#rocket_failure_errors = { path = "../../../rocket_failure/rocket_failure_errors" }
+rocket_failure_errors = "0.2"
 failure = "0.1"
-nom = "4.0"
+nom = "5.0"

sn0int-common/src/api.rs

@@ -1,3 +1,5 @@
+use crate::id::ModuleID;
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct WhoamiResponse {
     pub user: String,
@@ -29,6 +31,13 @@ pub struct ModuleInfoResponse {
     pub name: String,
     pub description: String,
     pub latest: Option<String>,
+    pub redirect: Option<ModuleID>,
+}
+
+impl ModuleInfoResponse {
+    pub fn canonical(&self) -> String {
+        format!("{}/{}", self.author, self.name)
+    }
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -46,3 +55,8 @@ impl SearchResponse {
         format!("{}/{}", self.author, self.name)
     }
 }
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct LatestResponse {
+    pub time: Option<u64>,
+}

sn0int-common/src/id.rs

@@ -1,38 +1,40 @@
 use crate::errors::*;
 use nom;
-use nom::types::CompleteStr;
 use serde::{de, Serialize, Serializer, Deserialize, Deserializer};
 use std::fmt;
 use std::result;
 use std::str::FromStr;
 
 
+#[inline(always)]
 fn valid_char(c: char) -> bool {
-    nom::is_alphanumeric(c as u8) || c == '-'
+    nom::character::is_alphanumeric(c as u8) || c == '-'
 }
 
 pub fn valid_name(name: &str) -> Result<()> {
-    if token(CompleteStr(name)).is_ok() {
+    if token(name).is_ok() {
         Ok(())
     } else {
         bail!("String contains invalid character")
     }
 }
 
-named!(module<CompleteStr, ModuleID>, do_parse!(
-    author: token   >>
-    tag!("/")       >>
-    name: token     >>
-    eof!()          >>
-    (
-        ModuleID {
-            author: author.to_string(),
-            name: name.to_string(),
-        }
-    )
-));
+fn module(s: &str) -> nom::IResult<&str, ModuleID> {
+    let (input, (author, _, name)) = nom::sequence::tuple((
+        token,
+        nom::bytes::complete::tag("/"),
+        token,
+    ))(s)?;
+    Ok((input, ModuleID {
+        author: author.to_string(),
+        name: name.to_string(),
+    }))
+}
 
-named!(token<CompleteStr, CompleteStr>, take_while1!(valid_char));
+#[inline]
+fn token(s: &str) -> nom::IResult<&str, &str> {
+    nom::bytes::complete::take_while1(valid_char)(s)
+}
 
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub struct ModuleID {
@@ -50,8 +52,11 @@ impl FromStr for ModuleID {
     type Err = Error;
 
     fn from_str(s: &str) -> Result<ModuleID> {
-        let (_, module) = module(CompleteStr(s))
+        let (trailing, module) = module(s)
             .map_err(|err| format_err!("Failed to parse module id: {:?}", err))?;
+        if !trailing.is_empty() {
+            bail!("Trailing data in module id");
+        }
         Ok(module)
     }
 }

sn0int-common/src/lib.rs

@@ -9,7 +9,7 @@ pub mod metadata;
 pub mod id;
 pub use crate::id::*;
 
-pub use rocket_failure::StrictApiResponse as ApiResponse;
+pub use rocket_failure_errors::StrictApiResponse as ApiResponse;
 
 #[cfg(test)]
 mod tests {

sn0int-common/src/metadata.rs

@@ -3,7 +3,7 @@ use crate::errors::*;
 use std::str::FromStr;
 
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum EntryType {
     Description,
     Version,
@@ -38,9 +38,36 @@ pub enum Source {
     Networks,
     Devices,
     Accounts(Option<String>),
+    Breaches,
+    Images,
+    Ports,
+    Netblocks,
+    CryptoAddrs(Option<String>),
     KeyRing(String),
 }
 
+impl Source {
+    pub fn group_as_str(&self) -> &'static str {
+        match self {
+            Source::Domains => "domains",
+            Source::Subdomains => "subdomains",
+            Source::IpAddrs => "ipaddrs",
+            Source::Urls => "urls",
+            Source::Emails => "emails",
+            Source::PhoneNumbers => "phonenumbers",
+            Source::Networks => "networks",
+            Source::Devices => "devices",
+            Source::Accounts(_) => "accounts",
+            Source::Breaches => "breaches",
+            Source::Images => "images",
+            Source::Ports => "ports",
+            Source::Netblocks => "netblocks",
+            Source::CryptoAddrs(_) => "cryptoaddrs",
+            Source::KeyRing(_) => "keyring",
+        }
+    }
+}
+
 impl FromStr for Source {
     type Err = Error;
 
@@ -62,6 +89,11 @@ impl FromStr for Source {
             ("networks", None) => Ok(Source::Networks),
             ("devices", None) => Ok(Source::Devices),
             ("accounts", param) => Ok(Source::Accounts(param.map(String::from))),
+            ("breaches", None) => Ok(Source::Breaches),
+            ("images", None) => Ok(Source::Images),
+            ("ports", None) => Ok(Source::Ports),
+            ("netblocks", None) => Ok(Source::Netblocks),
+            ("cryptoaddrs", param) => Ok(Source::CryptoAddrs(param.map(String::from))),
             ("keyring", Some(param)) => Ok(Source::KeyRing(param.to_string())),
             (x, Some(param)) => bail!("Unknown Source: {:?} ({:?})", x, param),
             (x, None) => bail!("Unknown Source: {:?}", x),

sn0int-registry/Cargo.toml

@@ -8,32 +8,31 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.5.0", path="sn0int-common" }
+sn0int-common = { version="0.10.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
-rocket_failure = { version = "0.1.2", features = ["with-rocket"] }
-rocket_contrib = { version = "0.4", features = ["handlebars_templates"] }
+#rocket_failure = { path = "../../rocket_failure" }
+rocket_failure = { version = "0.2" }
+rocket_contrib = { version = "0.4.1", features = ["handlebars_templates"] }
 
 diesel = { version = "1.3", features = ["postgres", "r2d2"] }
 diesel_migrations = { version = "1.3.0", features = ["postgres"] }
 diesel_full_text_search = "1.0.1"
 
 reqwest = "0.9.2"
-oauth2 = "2.0.0-alpha.2"
+oauth2 = "2.0.0"
 failure = "0.1"
 url = "1.0"
 log = "0.4"
 semver = "0.9.0"
 lazy_static = "1"
 blake2 = "0.8.0"
-hex = "0.3.1"
+hex = "0.4"
 maplit = "1.0.1"
+syntect = "3.3"
 
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 
-dotenv = "0.13"
-
-[patch.crates-io]
-rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }
-rocket_contrib = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }
+dotenv = "0.15"
+env_logger = "0.7"

sn0int-registry/Dockerfile

@@ -1,15 +1,17 @@
-FROM rust
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq llvm libclang-dev \
+    && rm -rf /var/lib/apt/lists/*
 RUN rustup install nightly
-WORKDIR /usr/src/sn0int-registry
+WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo +nightly build --release --verbose
+RUN cd sn0int-registry && cargo +nightly build --release --verbose
 RUN strip target/release/sn0int-registry
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libcurl3 libpq5 \
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libcurl4 libpq5 \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=0 /usr/src/sn0int-registry/target/release/sn0int-registry /usr/local/bin/sn0int-registry
-COPY templates /templates
+COPY --from=0 /usr/src/sn0int/target/release/sn0int-registry /usr/local/bin/sn0int-registry
+COPY sn0int-registry/templates /templates
 ENV ROCKET_ENV=prod \
     ROCKET_ADDRESS=0.0.0.0 \
     ROCKET_PORT=8000