Release v0.12.0

Add ports to database

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.9.0"
+version = "0.12.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -17,8 +17,8 @@ members = ["sn0int-registry/sn0int-common",
            "sn0int-registry"]
 
 [dependencies]
-sn0int-common = { version="0.4.0", path="sn0int-registry/sn0int-common" }
-rustyline = "3"
+sn0int-common = { version="0.7.0", path="sn0int-registry/sn0int-common" }
+rustyline = "4.0"
 log = "0.4"
 env_logger = "0.6"
 hlua-badtouch = "0.4"
@@ -32,10 +32,10 @@ publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
 chrono = { version = "0.4", features = ["serde"] }
-dirs = "1.0"
+dirs = "2.0"
 url = "1.7"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.8"
+chrootable-https = "0.10"
 base64 = "0.10"
 kuchiki = "0.7.2"
 serde_urlencoded = "0.5"
@@ -44,12 +44,12 @@ serde_derive = "1.0"
 serde_json = "1.0"
 crossbeam-channel = "0.3"
 ctrlc = "3.1"
-opener = "0.3.0"
+opener = "0.4"
 separator = "0.4"
 maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.2"
 regex = "1.0"
-toml = "0.4"
+toml = "0.5"
 maxminddb = "0.13"
 tar = "0.4.17"
 libflate = "0.1.14"
@@ -60,15 +60,37 @@ nom = "4.1.1"
 atty = "0.2"
 bufstream = "0.1.4"
 tokio = "0.1.14"
+semver = "0.9"
+bytes = "0.4"
+xml-rs = "0.8"
+bytesize = "1.0"
+
+digest = "0.8.0"
+hex = "0.3.1"
+bs58 = "0.2.2"
+blake2 = "0.8.0"
+md-5 = "0.8.0"
+sha-1 = "0.8.1"
+sha2 = "0.8.0"
+sha3 = "0.8.0"
+hmac = "0.7"
+
+image = "0.21"
+kamadak-exif = "0.3.1"
+walkdir = "2.2"
+nude = "0.1.0"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
-syscallz = "0.8"
-nix = "0.13"
+#syscallz = { path="../syscallz-rs" }
+syscallz = "0.11"
+nix = "0.14"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.3.1"
 unveil = "0.2.0"
 
 [dev-dependencies]
-boxxy = "0.8"
+#boxxy = { path = "../boxxy-rs" }
+boxxy = "0.10"
+tempfile = "3.0"

Dockerfile

@@ -1,13 +1,14 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM debian
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

ISSUE_TEMPLATE.md

@@ -0,0 +1,29 @@
+<!--
+Hello!
+
+If you want to report a bug we added some common questions below that help us analyse your issue faster.
+
+All of these are optional so feel free to remove anything that doesn't apply.
+-->
+
+please describe your issue here
+
+---
+
+## Versions
+
+- **rustc --version:**
+- **cargo --version:**
+- **sn0int --version:**
+- **uname -a:**
+
+## Environment
+
+- **Operating System/Distro:**
+- **Installed from (source/apt/pacman/brew/docker):**
+
+<!--
+Thank you!
+
+We'll try to respond as quickly as possible.
+-->

Makefile

@@ -19,3 +19,10 @@ update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt
 	get-iab -v -u http://standards-oui.ieee.org/iab/iab.txt -f data/ieee-iab.txt
 	rm -f data/ieee-*.txt.bak
+
+docs:
+	$(MAKE) -C docs html
+	contrib/html-toc2md.pl README.md docs/_build/html/index.html > README2.md
+	mv README2.md README.md
+
+.PHONY: check force-check test update docs

README.md

@@ -1,4 +1,4 @@
-# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs]
+# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
 [travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
 [travis]:       https://travis-ci.org/kpcyrd/sn0int
@@ -6,6 +6,12 @@
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
 [docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
+[irc-img]:      https://img.shields.io/badge/hackint-%23sn0int-blue.svg
+[irc]:          https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int
+[mastodon-img]: https://img.shields.io/badge/mastodon-chaos.social-blue.svg
+[mastodon]:     https://chaos.social/@sn0int
+[registry-img]: https://img.shields.io/website/https/sn0int.com.svg?label=registry
+[registry]:     https://sn0int.com/
 
 sn0int is a semi-automatic OSINT framework and package manager. It was built
 for IT security professionals and bug hunters to gather intelligence about a
@@ -15,36 +21,51 @@ unified format for followup investigations.
 
 Among other things, sn0int is currently able to:
 
-- Harvest subdomains from certificate transparency logs
-- Harvest subdomains from various passive dns logs
-- Sift through subdomain results for publicly accessible websites
-- Harvest emails from pgp keyservers
-- Enrich ip addresses with ASN and geoip info
-- Harvest subdomains from the wayback machine
+- Harvest subdomains from certificate transparency logs and passive dns
+- Enrich ip addresses with asn and geoip info
+- Harvest emails from pgp keyservers and whois
+- Discover compromised logins in breaches
+- Find somebody's profiles across the internet
+- Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
-- Bruteforce interesting urls
+- Harvest data and images from instagram profiles
+- Scan images for nudity
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
-and is fully opensource.  None of the investigations listed above are hardcoded
+and is fully opensource. None of the investigations listed above are hardcoded
 in the source, instead those are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
 you to ship updates for your modules on your own since you don't need to send a
 pull request.
 
-Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
+For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
 [![asciicast](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB.svg)](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB)
 
+## Installation
+
+Archlinux
+
+    pacman -S sn0int
+
+Mac OSX
+
+    brew install sn0int
+
+For everything else please have a look at the [detailed list][1].
+
+[1]: https://sn0int.readthedocs.io/en/latest/install.html
+
 ## Getting started
 
 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
-  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
-  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
-  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
-  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
+  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
+  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
+  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
 - [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
   - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
@@ -55,23 +76,46 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
 - [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
   - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
+- [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/structs.html#domains)
+  - [Subdomains](https://sn0int.readthedocs.io/en/latest/structs.html#subdomains)
+  - [IpAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#ipaddrs)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/structs.html#urls)
+  - [Emails](https://sn0int.readthedocs.io/en/latest/structs.html#emails)
+  - [Phonenumbers](https://sn0int.readthedocs.io/en/latest/structs.html#phonenumbers)
+  - [Devices](https://sn0int.readthedocs.io/en/latest/structs.html#devices)
+  - [Networks](https://sn0int.readthedocs.io/en/latest/structs.html#networks)
+  - [Accounts](https://sn0int.readthedocs.io/en/latest/structs.html#accounts)
+  - [Breaches](https://sn0int.readthedocs.io/en/latest/structs.html#breaches)
+  - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
+  - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
+  - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
+- [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
+- [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
+- [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
   - [Using access keys as source argument](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-as-source-argument)
 - [Configuration](https://sn0int.readthedocs.io/en/latest/config.html)
-  - [Configuring a proxy](https://sn0int.readthedocs.io/en/latest/config.html#configuring-a-proxy)
+  - [\[core\]](https://sn0int.readthedocs.io/en/latest/config.html#core)
+  - [\[namespaces\]](https://sn0int.readthedocs.io/en/latest/config.html#namespaces)
+  - [\[network\]](https://sn0int.readthedocs.io/en/latest/config.html#network)
 - [Sandbox](https://sn0int.readthedocs.io/en/latest/sandbox.html)
   - [Linux](https://sn0int.readthedocs.io/en/latest/sandbox.html#linux)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/sandbox.html#openbsd)
   - [IPC Protocol](https://sn0int.readthedocs.io/en/latest/sandbox.html#ipc-protocol)
   - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
+  - [Diagnosing a sandbox failure](https://sn0int.readthedocs.io/en/latest/sandbox.html#diagnosing-a-sandbox-failure)
 - [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
+  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
   - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
+  - [create_blob](https://sn0int.readthedocs.io/en/latest/reference.html#create-blob)
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
@@ -79,25 +123,40 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
   - [error](https://sn0int.readthedocs.io/en/latest/reference.html#error)
-  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
   - [geoip_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#geoip-lookup)
+  - [hex](https://sn0int.readthedocs.io/en/latest/reference.html#hex)
+  - [hmac_md5](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-md5)
+  - [hmac_sha1](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha1)
+  - [hmac_sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-256)
+  - [hmac_sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-512)
+  - [hmac_sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-256)
+  - [hmac_sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-512)
   - [html_select](https://sn0int.readthedocs.io/en/latest/reference.html#html-select)
   - [html_select_list](https://sn0int.readthedocs.io/en/latest/reference.html#html-select-list)
   - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
   - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
   - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
+  - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
+  - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [sha1](https://sn0int.readthedocs.io/en/latest/reference.html#sha1)
+  - [sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-256)
+  - [sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-512)
+  - [sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-256)
+  - [sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-512)
   - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
@@ -113,6 +172,9 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
+  - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
   - [url_escape](https://sn0int.readthedocs.io/en/latest/reference.html#url-escape)
@@ -120,7 +182,28 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [url_parse](https://sn0int.readthedocs.io/en/latest/reference.html#url-parse)
   - [url_unescape](https://sn0int.readthedocs.io/en/latest/reference.html#url-unescape)
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
+  - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
+  - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
+  - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
+  - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)
+
+## Rationale
+
+This tool was written for companies to help them understand their attack
+surface from a blackbox point of view. It's often difficult to understand that
+something is easier to discover than some people assume, putting them at risk
+of false security.
+
+It's also designed to be useful for red team assessments and bug bounties,
+which also help companies to identify weaknesses that could result in a
+compromise.
+
+Some functionality was written to do the same thing for individuals to raise
+awareness about personal attack surface, privacy and how much data is publicly
+available. These issues are often out of scope in bug bounties and sometimes by
+design. We believe that blaming the user is the wrong approach and these issues
+should be addressed at the root cause by the people designing those systems.
 
 ## License
 

ci/boxxy_stage1.txt

@@ -1,3 +1,5 @@
 stage1
 ls
+echo checkpoint1
+id
 echo sandbox fail

ci/integration.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+import subprocess
+from subprocess import DEVNULL, PIPE
+import tempfile
+import json
+import sys
+
+
+def _sn0int(tempdir, binary, args, piped_stdout=False):
+    return subprocess.Popen(
+        ['/usr/bin/env', 'HOME='+tempdir, binary] + args,
+        stdin=PIPE,
+        stdout=PIPE if piped_stdout else None,
+    )
+
+
+def sn0int(tempdir, binary, cmds):
+    p = _sn0int(tempdir, binary, [])
+    for cmd in cmds:
+        p.stdin.write((cmd + '\n').encode('utf-8'))
+    p.communicate()
+    if p.returncode != 0:
+        raise Exception('process failed')
+
+
+def sn0int_select(tempdir, binary, query):
+    p = _sn0int(tempdir, binary, ['select', '--json'] + query, piped_stdout=True)
+    stdout, _ = p.communicate()
+    lines = filter(None, stdout.decode('utf-8').split('\n'))
+    return [json.loads(x) for x in lines]
+
+
+def main(tempdir, binary):
+    print('[*] setting up workspace')
+    sn0int(tempdir, binary, [])
+
+    print('[*] adding domain')
+    sn0int(tempdir, binary, [
+        'add domain',
+        'example.com',
+        'select domains',
+    ])
+
+    print('[*] testing db for domain')
+    domains = sn0int_select(tempdir, binary, ['domains'])
+    assert domains == [{'id': 1, 'value': 'example.com', 'unscoped': False}]
+
+    print('[*] installing modules')
+    sn0int(tempdir, binary, [
+        'mod install kpcyrd/ctlogs',
+        'mod install kpcyrd/dns-resolve',
+        'mod install kpcyrd/url-scan',
+        'mod install kpcyrd/geoip',
+    ])
+
+    print('[*] running ctlogs')
+    sn0int(tempdir, binary, [
+        'use ctlogs',
+        'run',
+        'select subdomains',
+    ])
+
+    print('[*] testing db for subdomains')
+    subdomains = sn0int_select(tempdir, binary, ['subdomains'])
+    assert {x['value'] for x in subdomains} == {
+        'www.example.com',
+        'm.example.com',
+        'dev.example.com',
+        'products.example.com',
+        'support.example.com',
+    }
+
+    print('[*] running dns-resolve')
+    sn0int(tempdir, binary, [
+        'use dns-resolve',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs')
+    ipaddrs = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert len(ipaddrs) >= 1
+
+    print('[*] running url-scan')
+    sn0int(tempdir, binary, [
+        'use url-scan',
+        'run',
+        'select urls',
+    ])
+
+    print('[*] testing db for urls')
+    urls = sn0int_select(tempdir, binary, ['urls'])
+    assert {(x['value'], x['status']) for x in urls} == {
+        ('http://www.example.com/', 200),
+        ('https://www.example.com/', 200),
+    }
+
+    print('[*] running geoip')
+    sn0int(tempdir, binary, [
+        'use geoip',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs again')
+    ipaddrs2 = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert ipaddrs != ipaddrs2
+
+    print('')
+    print('\t###########')
+    print('\t# SUCCESS #')
+    print('\t###########')
+    print('')
+
+
+if __name__ == '__main__':
+    try:
+        binary = sys.argv[1]
+    except IndexError:
+        print('Usage: %s target/release/sn0int' % sys.argv[0])
+    else:
+        with tempfile.TemporaryDirectory(prefix='sn0int-') as tempdir:
+            main(tempdir, binary)

contrib/html-toc2md.pl

@@ -0,0 +1,39 @@
+#!/usr/bin/env perl
+use strict; use warnings;
+
+my ($readme, $toc) = @ARGV;
+defined $readme or die 'missing readme path';
+defined $toc or die 'missing toc path';
+
+open(my $r, "<$readme") or die 'failed to open readme';
+open(my $t, "<$toc") or die 'failed to open toc';
+
+my $re = qr/^\s*- \[.+\]\(https:\/\/sn0int.readthedocs.io\/en\/.+\)$/;
+
+# pass through start of readme
+while (<$r>) {
+    last if ($_ =~ $re);
+    print $_;
+}
+
+# skip toc
+while (<$r>) {
+    last unless ($_ =~ $re);
+}
+
+# generate new toc
+while (my $line = <$t>) {
+    if ($line =~ /toctree-l(\d).*href="([^"]+)">(.+)<\/a/) {
+        my $space = $1;
+        my $section = $2;
+        my $label = $3;
+        $label =~ s/([\[\]])/\\$1/g;
+        print $space==2?"  ":"", "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
+    }
+}
+print;
+
+# pass through end of readme
+while (<$r>) {
+    print $_;
+}

contrib/html-toc2md.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-perl -n -e '/toctree-l(\d).*href="([^"]+)">(.+)<\/a/ && print $1==2?"  ":"", "- [$3](https://sn0int.readthedocs.io/en/latest/$2)\n"' < docs/_build/html/index.html

docs/config.rst

@@ -1,17 +1,54 @@
 Configuration
 =============
 
-This file documents the config file at ``~/.config/sn0int.toml``. By default
-this file does not exist and a default configuration is used instead.
+This section documents the config file. By default this file does not exist and
+a default configuration is used instead.
 
-Configuring a proxy
--------------------
+Linux/BSD
+    ``~/.config/sn0int.toml``
+
+OSX
+    ``~/Library/Preferences/sn0int.toml``
+
+Windows
+    ``%APPDATA%/sn0int.toml``
+
+[core]
+------
+
+``registry``
+    Configure the registry you want to use. Defaults to ``https://sn0int.com``.
+``no-autoupdate``
+    sn0int is going to check if your modules are outdated during startout once
+    a week. Set this option to ``true`` to disable this.
+
+[namespaces]
+------------------
+
+By default sn0int modules are assumed to be installed from the registry. You
+may want to keep a local directory with private modules, especially during
+development. You can configure a folder that contains modules that aren't
+managed by sn0int by adding a namespace section to the config file::
+
+    [namespaces]
+    foo = "/opt/sn0int/foo"
+    bar = "~/repos/a/b/c/sn0int-modules"
+
+This is going to load modules from these two folders and register them in the
+``foo`` and ``bar`` namespace.
+
+Note that sn0int is also going to assume that symlinks in
+``~/.local/share/sn0int/modules`` and folders containing a ``.git`` folder are
+externally managed.
+
+[network]
+---------
 
 To enable a proxy, add the following to your config file::
 
     [network]
     proxy = "127.0.0.1:9050"
 
-This forces everything through tor and restricts all other functions that
-depend on the network. For example the ``dns`` function is fully disabled if a
-proxy is configured.
+This forces everything through tor (or any other socks5 proxy) and restricts
+all other functions that depend on the network. For example the ``dns``
+function is fully disabled if a proxy is configured.

docs/index.rst

@@ -39,6 +39,7 @@ Getting Started
    usage
    scripting
    database
+   structs
    keyring
    config
    sandbox

docs/install.rst

@@ -10,22 +10,27 @@ Archlinux
 
     $ pacman -S sn0int
 
+Mac OSX
+-------
+
+.. code-block:: bash
+
+    $ brew install sn0int
+
 Debian/Ubuntu/Kali
 ------------------
 
-.. code-block:: bash
+Note that debian `doesn't ship the geoip2-database
+<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
+download them automatically during the first run.
 
-    $ apt install libsqlite3-dev libseccomp-dev
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
-
-Alpine
-------
+Using rust+cargo from the repos might work for you, but we only officially
+support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
+at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
+    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .
@@ -37,21 +42,22 @@ Docker
 
     $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
 
+Alpine
+------
+
+.. code-block:: bash
+
+    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f --path .
+
 OpenBSD
 -------
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
-
-Mac OSX
--------
-
-.. code-block:: bash
-
+    $ pkg_add git cargo sqlite3 geolite2-city geolite2-asn
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .

docs/keyring.rst

@@ -52,8 +52,8 @@ If the user granted us access to those keys we can read them with ``keyring``:
 .. code-block:: lua
 
     creds = keyring('aws')
-    print(creds[1]['accesskey'])
-    print(creds[1]['secretkey'])
+    debug(creds[1]['access_key'])
+    debug(creds[1]['secret_key'])
 
 This returns a list of all keys in that namespace. Any empty list is returned
 if the user doesn't have any keys in that namespace.

docs/reference.rst

@@ -1,6 +1,17 @@
 Function reference
 ==================
 
+asn_lookup
+----------
+
+Run an ASN lookup for a given ip address. The function returns ``asn`` and
+``as_org``. This function may fail.
+
+.. code-block:: lua
+
+    lookup = asn_lookup('1.1.1.1')
+    if last_err() then return end
+
 clear_err
 ---------
 
@@ -13,6 +24,18 @@ Clear the last recorded error from the internal state. See also last_err_.
         clear_err()
     end
 
+create_blob
+-----------
+
+Push a byte array into persistent blob storage. This allows passing those bytes
+to functions operating on blob storage. Returns a blob identifier that is
+deterministic based on the blob content. Blobs are immutable.
+
+.. code-block:: lua
+
+    blob = create_blob("some bytes")
+    debug(blob)
+
 datetime
 --------
 
@@ -23,6 +46,10 @@ for ``DATETIME`` database fields.
 
     now = datetime()
 
+.. note::
+    This format is sn0int specific, to get the current time for scripting use
+    time_unix_ instead.
+
 db_add
 ------
 
@@ -123,17 +150,6 @@ Log an error to the terminal.
 
     error('ohai')
 
-asn_lookup
-----------
-
-Run an ASN lookup for a given ip address. The function returns ``asn`` and
-``as_org``. This function may fail.
-
-.. code-block:: lua
-
-    lookup = asn_lookup('1.1.1.1')
-    if last_err() then return end
-
 geoip_lookup
 ------------
 
@@ -154,6 +170,69 @@ This function may fail.
     lookup = geoip_lookup('1.1.1.1')
     if last_err() then return end
 
+hex
+---
+
+Hex encode a list of bytes.
+
+.. code-block:: lua
+
+    hex("\x6F\x68\x61\x69\x0A\x00")
+
+hmac_md5
+--------
+
+Calculate an hmac with md5. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_md5("secret", "my authenticated message")
+
+hmac_sha1
+---------
+
+Calculate an hmac with sha1. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha1("secret", "my authenticated message")
+
+hmac_sha2_256
+-------------
+
+Calculate an hmac with sha2_256. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha2_256("secret", "my authenticated message")
+
+hmac_sha2_512
+-------------
+
+Calculate an hmac with sha2_512. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha2_512("secret", "my authenticated message")
+
+hmac_sha3_256
+-------------
+
+Calculate an hmac with sha3_256. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha3_256("secret", "my authenticated message")
+
+hmac_sha3_512
+-------------
+
+Calculate an hmac with sha3_512. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha3_512("secret", "my authenticated message")
+
 html_select
 -----------
 
@@ -210,6 +289,9 @@ options are set. The following options are available:
   The request body that should be form encoded.
 ``body``
   The raw request body as string.
+``into_blob``
+  If true, the response body is stored in blob storage and a blob reference is
+  returned as ``blob`` instead of the full body.
 
 This function may fail.
 
@@ -237,6 +319,9 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``blob``
+  If ``into_blob`` was enabled for the request the body is downloaded into blob
+  storage with a reference to the body in this field.
 
 .. code-block:: lua
 
@@ -250,6 +335,41 @@ the following keys:
     if last_err() then return end
     if resp["status"] ~= 200 then return "invalid status code" end
 
+img_load
+--------
+
+Attempt to decode a blob as an image and return some basic metadata like the
+mime type, height and width.
+
+.. code-block:: lua
+
+    img = img_load(blob)
+    if last_err() then return end
+    debug(img)
+
+img_exif
+--------
+
+Extract exif metadata from an image.
+
+.. code-block:: lua
+
+    exif = img_exif(blob)
+    if last_err() then return end
+    debug(exif)
+
+img_nudity
+----------
+
+Classify an image for nudity. The score goes from 0 to 2. A score above 1 means
+nudity has been detected.
+
+.. code-block:: lua
+
+    nudity = img_nudity(blob)
+    if last_err() then return end
+    debug(nudity)
+
 info
 ----
 
@@ -318,6 +438,15 @@ otherwise.
         return
     end
 
+md5
+---
+
+Hash a byte array with md5 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(md5("\x00\xff"))
+
 pgp_pubkey
 ----------
 
@@ -390,12 +519,12 @@ psl_domain_from_dns_name
 ------------------------
 
 Returns the parent domain according to the public suffix list. For
-``www.a.b.c.d.example.com`` this is going to be ``example.com``.
+``www.a.b.c.d.example.co.uk`` this is going to be ``example.co.uk``.
 
 .. code-block:: lua
 
-    domain = psl_domain_from_dns_name('www.a.b.c.d.example.com')
-    print(domain == 'example.com')
+    domain = psl_domain_from_dns_name('www.a.b.c.d.example.co.uk')
+    print(domain == 'example.co.uk')
 
 regex_find
 ----------
@@ -430,6 +559,51 @@ Same as regex_find_, but returns all matches.
     print(m[3][1] == 'ef')
     print(m[3][2] == 'f')
 
+sha1
+----
+
+Hash a byte array with sha1 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha1("\x00\xff"))
+
+sha2_256
+--------
+
+Hash a byte array with sha2_256 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha2_256("\x00\xff"))
+
+sha2_512
+--------
+
+Hash a byte array with sha2_512 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha2_512("\x00\xff"))
+
+sha3_256
+--------
+
+Hash a byte array with sha3_256 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha3_256("\x00\xff"))
+
+sha3_512
+--------
+
+Hash a byte array with sha3_512 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha3_512("\x00\xff"))
+
 sleep
 -----
 
@@ -571,6 +745,36 @@ Read a line from stdin. The final newline is not removed.
 .. note::
    This only works with `sn0int run --stdin`.
 
+strftime
+--------
+
+Format a timestamp generated with time_unix_ into a date, see `strftime rules`_.
+
+.. code-block:: lua
+
+    t = strftime('%d/%m/%Y %H:%M', 1558584994)
+
+strptime
+--------
+
+Parse a date into a unix timestamp, see `strftime rules`_.
+
+.. code-block:: lua
+
+    t = strptime('%d/%m/%Y %H:%M', '23/05/2019 04:16')
+
+.. _strftime rules: https://docs.rs/chrono/0.4.6/chrono/format/strftime/index.html
+
+time_unix
+---------
+
+Get the current time as seconds since ``January 1, 1970 0:00:00 UTC``, also
+known as UNIX timestamp. This timestamp can be formated using strftime_.
+
+.. code-block:: lua
+
+    now = time_unix()
+
 url_decode
 ----------
 
@@ -660,6 +864,27 @@ Decodes a list of bytes/numbers into a string. This function might fail.
     if last_err() then return end
     print(x == 'AAAA')
 
+warn
+----
+
+Log a warning to the terminal.
+
+.. code-block:: lua
+
+    warn('ohai')
+
+warn_once
+---------
+
+Log a warning to the terminal once. This can be used to print a warning to the
+user without printing the same warning for each struct we're processing during
+a ``run`` execution.
+
+.. code-block:: lua
+
+    warn_once('ohai')
+    warn_once('ohai')
+
 x509_parse_pem
 --------------
 
@@ -694,3 +919,35 @@ Parse a pem encoded certificate. This function might fail.
     ]])
     if last_err() then return end
     print(x)
+
+xml_decode
+----------
+
+Decode a lua value from an xml document.
+
+.. code-block:: lua
+
+    x = xml_decode('<body><foo fizz="buzz">bar</foo></body>')
+    if last_err() then return end
+
+    body = x['children'][1]
+    foo = body['children'][1]
+
+    print(foo['attrs']['fizz'])
+    print(foo['text'])
+
+xml_named
+---------
+
+Get a named child element from a parent element.
+
+.. code-block:: lua
+
+    x = xml_decode('<body><foo fizz="buzz">bar</foo></body>')
+    if last_err() then return end
+
+    body = x['children'][1]
+    foo = xml_named(body, 'foo')
+    if foo ~= nil then
+        print(foo)
+    end

docs/sandbox.rst

@@ -111,3 +111,37 @@ There are some limitations that you should be aware:
   means scripts have access to your local network, the internet and also your
   localhost loopback interface.
 - If chroot is unavailable an attacker could connect to unix domain sockets.
+
+Diagnosing a sandbox failure
+----------------------------
+
+You might experience a sandbox failure, especially on architectures that are
+less popular. This usually looks like this::
+
+    [sn0int][example][kpcyrd/ctlogs] > run
+    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [+] Finished kpcyrd/ctlogs (1 errors)
+
+A module that never finishes could also mean an IO thread inside the worker got
+killed by the sandbox.
+
+You can try to diagnose this yourself with strace::
+
+    strace -f sn0int run -vv ctlogs 2>&1 | tee strace.log
+
+Open ``strace.log``, look out for syscalls that didn't return by searching for
+``= ?`` and ignore calls to exit and similar. You are looking for something
+like this::
+
+    seccomp(SECCOMP_SET_MODE_FILTER, 0, {len=48, filter=0xdd59094e490}) = 0
+    write(1, "[+] activated!\n", 15[+] activated!
+    )        = 15
+    getresuid( <unfinished ...>)            = ?
+    +++ killed by SIGSYS (core dumped) +++
+
+This would indicate a call to ``getresuid`` which was not allowed by the
+seccomp filter.
+
+If you don't want to diagnose this yourself open a new bug report with as much
+information as possible, specifically which distro, which release and which
+architecture you're using.

docs/scripting.rst

@@ -13,13 +13,29 @@ free to change that to something else::
 
     $ git init ~/repos/sn0int-modules
     $ cd ~/repos/sn0int-modules
-    $ ln -s "$PWD" ~/.local/share/sn0int/modules/$YOUR_GITHUB_NAME
+
+We need to add this folder to the sn0int config file so it's correctly detected
+when starting sn0int. Open the `config file <config.html>`_ in your prefered
+editor. Note that the file does not exist by default and the path is different
+depending on your operating system. On linux you would open the config file
+with::
+
+    $ vim ~/.config/sn0int.toml
+
+Add the following::
+
+    [namespaces]
+    your_github_name = "~/repos/sn0int-modules"
 
 Every module we're adding to ``~/repos/sn0int-modules`` is now going to be
 picked up by sn0int.
 
-Let's add our first module by opening ``~/repos/sn0int-modules/first.lua``.
-There's a bit of boilerplate that every module needs to load successfully:
+Make sure you're still in the right folder and add your first module::
+
+    sn0int new first.lua
+
+This is going to generate some boilerplate for you that every module needs to
+load successfully. Afterwards we can edit it like this:
 
 .. code-block:: lua
 
@@ -188,3 +204,40 @@ your identity.
 Afterwards publish your module with::
 
     sn0int publish ./first.lua
+
+Reading data from stdin
+-----------------------
+
+Sometimes you need to read data that can't be easily accessed from within the
+sandbox, like output of other programms or file content. In that case you can
+write a module that reads from stdin:
+
+.. code-block:: lua
+
+    -- Description: Read from stdin
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+
+    function run()
+        while true do
+            x = stdin_readline()
+            if x == nil then
+                break
+            end
+            info(x)
+        end
+    end
+
+Write it to a file and run it like this::
+
+    % echo hello | sn0int run --stdin -vvf stdin.lua
+    [*] anonymous/stdin                                   : "hello\n"
+    [+] Finished anonymous/stdin
+    %
+
+This is going to read one line at a time and allows you to process it with
+regular expressions and add data to the database.
+
+.. note::
+   If you get an error like ``Failed to read stdin: "stdin is unavailable"``
+   make sure the ``--stdin`` flag is set.

docs/structs.rst

@@ -0,0 +1,274 @@
+Structs
+=======
+
+This section describes all supported structs in depth. Please refer to this
+section if in doubt about the correct usage of fields to ensure
+interoperability between modules.
+
+Domains
+-------
+
+Represents a registerable domain as defined by the `public suffix list
+<https://publicsuffix.org/>`_. If in doubt check `psl_domain_from_dns_name
+<reference.html#psl-domain-from-dns-name>`_.
+
+``value``
+  The domain name, like ``example.co.uk``.
+
+Subdomains
+----------
+
+A subdomain of a `domain <#domains>`_. The depth is arbitrary, so
+``foo.example.co.uk`` and ``foo.bar.example.co.uk`` are both valid subdomains
+of ``example.co.uk``.
+
+``value``
+  The subdomain, like ``foo.bar.example.co.uk``.
+``domain_id``
+  The numeric id of a domain struct.
+``resolvable``
+  Whether the subdomain can be resolved to a A/AAAA record. nil if unknown.
+
+IpAddrs
+-------
+
+An ip address. Note that most of these fields are geoip related and an
+approximation instead of an actual location.
+
+``value``
+    The ip address.
+``family``
+    The address family of the ip address, either ``4`` or ``6``.
+``continent``
+    The continent associated with this ip address.
+``continent_code``
+    The continent code of the ``continent`` field, eg ``NA``.
+``country``
+    The country associated with this ip address.
+``country_code``
+    The country code of the ``country`` field, eg ``US``.
+``city``
+    The city associated with this ip address.
+``latitude``
+    Latitude associated with this ip address.
+``longitude``
+    Longitude associated with this ip address.
+``asn``
+    The number of the autonomous system this ip belongs to.
+``as_org``
+    The organization of the autonomous system this ip belongs to.
+``description``
+    This field is sn0int internal if we have additional information about this
+    ip address, for example technical identifiers from aws.
+``reverse_dns``
+    The reverse dns name setup for this ip address.
+
+URLs
+----
+
+``subdomain_id``
+    The numeric id of a subdomain struct.
+``value``
+    The url, including a schema, hostname and path.
+``status``
+    The http status code, like ``200``.
+``body``
+    The raw response body. This can be any mime type.
+``online``
+    Whether or not the url gives a http response (even if it's an error).
+``title``
+    The parsed ``<title>`` of the page, if available.
+``redirect``
+    If the server replied with a redirect, this is the url it redirected to.
+
+Emails
+------
+
+``value``
+    The email address.
+``displayname``
+    The display name of a given email address: ``this is the name <foo@example.com>``.
+``valid``
+    Whether that email address is valid or has been disabled.
+
+Phonenumbers
+------------
+
+``value``
+    The phone number in E.164 format (+491234567)
+``name``
+    An alias we can assign to this phone number. This alias is sn0int internal.
+``valid``
+    Whether the number is assigned to a customer.
+``last_online``
+    The last time this number has been online.
+``country``
+    The country this number is associated with.
+``carrier``
+    The name of the carrier this numer is registered with.
+``line``
+    The type of the phone number, can be ``landline``, ``mobile`` or ``voip``.
+``is_ported``
+    Whether this number has been ported to a different carrier.
+``last_ported``
+    The last time this number has been ported.
+``caller_name``
+    The name of the owner of the phone number.
+``caller_type``
+    The type of caller, eg ``business`` or ``consumer``.
+
+Devices
+-------
+
+``value``
+    The devices mac address or another identifier if needed.
+``name``
+    An alias we can assign to this device. This alias is sn0int internal.
+``hostname``
+    The hostname configured on the device.
+``vendor``
+    The hardware vendor of the device. This is usually derived from the mac
+    address.
+``last_seen``
+    The last time we've observed the device somewhere.
+
+Networks
+--------
+
+A wired or wireless network at a specific location that a device could be
+connected to.
+
+``value``
+    The network name. This can be an ssid or any other identifier but should be
+    unique.
+``latitude``
+    Latitude of the networks location.
+``longitude``
+    Longitude of the networks location.
+
+Accounts
+--------
+
+A users account or profile on a webservice, like github or instagram.
+
+``service``
+    The identifier of the service/website. It's recommended to use the websites
+    domain for this as defined in `Domains`_.
+``username``
+    The users unique identifier, like the login name. If the login name is not
+    known or the system doesn't use login names, use the email address instead.
+``displayname``
+    The users display name. This name is often not unique and may contain the
+    users real name.
+``email``
+    The email address associated with the account.
+``url``
+    The url of the public profile if available.
+``last_seen``
+    The last time this account has been active/online.
+
+Breaches
+--------
+
+Either a breach of a specific website, a breach compilation or a breach
+notification service.
+
+``value``
+    The name of the breach, breach compilation or notification service.
+
+Images
+------
+
+``value``
+    The id that identifies the blob. This id is deterministic based on file
+    content.
+``filename``
+    This field is used if we have a well known filename for the content.
+``mime``
+    The image mimetype, like ``image/png`` or ``image/jpeg``.
+``width``
+    The width of the image.
+``height``
+    The height of the image.
+``created``
+    The date and time this image has been taken.
+``latitude``
+    Latitude this picture has been taken.
+``longitude``
+    Longitude this picture has been taken.
+``nudity``
+    A score that classifies nudity in this picture. The score goes from 0 to 2
+    and is commonly calculated with ``img_nudity``. A score above 1 means
+    nudity has been detected.
+``ahash``
+    The Mean (aHash) perceptual hash.
+``dhash``
+    The Gradient (dHash) perceptual hash.
+``phash``
+    The DCT (pHash) perceptual hash.
+
+Ports
+-----
+
+The status of a port on an ip address.
+
+``ip_addr_id``
+    The numeric id of an ipaddr struct.
+``ip_addr``
+    The actual ipaddr.
+``port``
+    The port number.
+``status``
+    The status of the port, either ``open`` or ``closed``.
+``banner``
+    The service banner we discovered on this port.
+``service``
+    The service that is running on this port.
+``version``
+    The version of the service running on this port.
+
+Relations
+---------
+
+Relations are linking two structs together. The link may contain additional information.
+
+subdomain_ipaddr
+~~~~~~~~~~~~~~~~
+
+Links an ip address to a subdomain.
+
+``subdomain_id``
+    The numeric id of a subdomain struct.
+``ip_addr_id``
+    The numeric id of an ip addr struct.
+
+network_device
+~~~~~~~~~~~~~~
+
+Links a device to a network. This is commonly used with ``db_add_ttl`` so the
+link automatically expires. This is frequently used to monitor networks for
+known and unknown devices.
+
+``network_id``
+    The numeric id of a network struct.
+``device_id``
+    The numeric id of a device struct.
+``ipaddr``
+    The ip address assigned to the device.
+``last_seen``
+    The last time we've seen the device on that network.
+
+breach_email
+~~~~~~~~~~~~
+
+Links an email to a breach. If we know the password as well we can add it to
+the link. If we don't know the password we can leave it blank and fill it
+later. An email can be linked to a breach multiple times with different
+passwords. There is a special upserting logic in place to support this.
+
+``breach_id``
+    The numeric id of a breach struct.
+``email_id``
+    The numeric id of an email struct.
+``password``
+    The password for that email in the breach.

migrations/2019-02-03-123424_accounts/down.sql

@@ -0,0 +1 @@
+DROP TABLE accounts;

migrations/2019-02-03-123424_accounts/up.sql

@@ -0,0 +1,12 @@
+CREATE TABLE accounts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);

migrations/2019-02-11-011316_breaches/down.sql

@@ -0,0 +1,2 @@
+DROP TABLE breach_emails;
+DROP TABLE breaches;

migrations/2019-02-11-011316_breaches/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE breaches (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT breach_unique UNIQUE (value)
+);
+
+CREATE TABLE breach_emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    breach_id INTEGER NOT NULL,
+    email_id INTEGER NOT NULL,
+    password VARCHAR,
+    FOREIGN KEY(breach_id) REFERENCES breaches(id) ON DELETE CASCADE,
+    FOREIGN KEY(email_id) REFERENCES emails(id) ON DELETE CASCADE,
+    CONSTRAINT breach_emails_unique UNIQUE (breach_id, email_id, password)
+);

migrations/2019-03-08-060037_images/down.sql

@@ -0,0 +1 @@
+DROP TABLE images;

migrations/2019-03-08-060037_images/up.sql

@@ -0,0 +1,21 @@
+CREATE TABLE images (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+
+    filename VARCHAR,
+    mime VARCHAR,
+    width INT,
+    height INT,
+    created DATETIME,
+
+    latitude FLOAT,
+    longitude FLOAT,
+
+    nudity FLOAT,
+    ahash VARCHAR,
+    dhash VARCHAR,
+    phash VARCHAR,
+
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT image_unique UNIQUE (value)
+);

migrations/2019-03-24-195306_email-names/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _emails_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO _emails_new (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM emails;
+
+DROP TABLE emails;
+ALTER TABLE _emails_new RENAME TO emails;
+
+PRAGMA foreign_keys=on;

migrations/2019-03-24-195306_email-names/up.sql

@@ -0,0 +1,2 @@
+-- Your SQL goes here
+ALTER TABLE emails ADD COLUMN displayname VARCHAR;

migrations/2019-05-30-142142_ports/down.sql

@@ -0,0 +1 @@
+DROP TABLE ports;

migrations/2019-05-30-142142_ports/up.sql

@@ -0,0 +1,17 @@
+CREATE TABLE ports (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);

modules/dev/apex-domain.lua

@@ -0,0 +1,11 @@
+-- Description: Create subdomain entries for apex domains
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    db_add('subdomain', {
+        domain_id=arg['id'],
+        value=arg['value'],
+    })
+end

modules/dev/arp-scan.lua

@@ -1,5 +1,5 @@
 -- Description: Parse arp-scan output
--- Version: 0.2.0
+-- Version: 0.3.0
 -- License: GPL-3.0
 
 -- sudo arp-scan -qglI wlp3s0

modules/dev/axfr.lua

@@ -1,10 +1,10 @@
 -- Description: Try a zone transfer for subdomains
--- Version: 0.2.0
+-- Version: 0.3.0
 -- Source: domains
 -- License: GPL-3.0
 
 function strip_root_dot(name)
-    m = regex_find("(.+)\\.$", name)
+    local m = regex_find("(.+)\\.$", name)
     if last_err() then return end
 
     if m == nil then
@@ -15,21 +15,19 @@ function strip_root_dot(name)
 end
 
 function add_pointer(name)
-    local domain, domain_id, subdomain_id
-
     -- select psl+1
-    domain = psl_domain_from_dns_name(name)
+    local domain = psl_domain_from_dns_name(name)
     if last_err() then return end
 
     -- add domain
-    domain_id = db_add('domain', {
+    local domain_id = db_add('domain', {
         value=domain,
     })
     if last_err() then return end
     if domain_id == nil then return end
 
     -- add subdomain
-    subdomain_id = db_add('subdomain', {
+    local subdomain_id = db_add('subdomain', {
         domain_id=domain_id,
         value=name,
     })
@@ -37,26 +35,24 @@ function add_pointer(name)
 end
 
 function iter_axfr(zone, arg)
-    local name, r, m, domain
-
     debug(arg)
 
-    name = arg[1]
-    r = arg[2]
+    local name = arg[1]
+    local r = arg[2]
 
     -- select psl+1
-    domain = psl_domain_from_dns_name(name)
+    local domain = psl_domain_from_dns_name(name)
     if last_err() then return end
 
     -- add domain
-    domain_id = db_add('domain', {
+    local domain_id = db_add('domain', {
         value=domain,
     })
     if last_err() then return end
     if domain_id == nil then return end
 
     -- add subdomain
-    subdomain_id = db_add('subdomain', {
+    local subdomain_id = db_add('subdomain', {
         domain_id=domain_id,
         value=name,
     })
@@ -65,7 +61,7 @@ function iter_axfr(zone, arg)
     -- this is a A record
     if r['A'] ~= nil then
         -- add the name and ip
-        ipaddr_id = db_add('ipaddr', {
+        local ipaddr_id = db_add('ipaddr', {
             family='4',
             value=r['A'],
         })
@@ -98,12 +94,10 @@ function iter_axfr(zone, arg)
 end
 
 function iter_a(zone, arg)
-    local i, records, r
-
     if arg == nil then return end
 
     debug('nameserver: ' .. arg)
-    records = dns(zone, {
+    local records = dns(zone, {
         record='AXFR',
         nameserver=arg .. ':53',
         tcp=true,
@@ -112,48 +106,40 @@ function iter_a(zone, arg)
     if records['error'] ~= nil then return end
     records = records['answers']
 
-    i = 1
-    while records[i] ~= nil do
+    for i=1, #records do
         iter_axfr(zone, records[i])
         if last_err() then return end
-        i = i+1
     end
 end
 
 function iter_ns(zone, arg)
-    local i, records, r
-
     if arg == nil then return end
 
-    records = dns(arg, {
+    local records = dns(arg, {
         record='A',
     })
     if last_err() then return end
     if records['error'] ~= nil then return end
     records = records['answers']
 
-    i = 1
-    while records[i] ~= nil do
+    for i=1, #records do
         r = records[i][2]
         iter_a(zone, r['A'])
         if last_err() then return end
-        i = i+1
     end
 end
 
 function run(arg)
-    records = dns(arg['value'], {
+    local records = dns(arg['value'], {
         record='NS',
     })
     if last_err() then return end
     if records['error'] ~= nil then return end
     records = records['answers']
 
-    i = 1
-    while records[i] ~= nil do
-        r = records[i][2]
+    for i=1, #records do
+        local r = records[i][2]
         iter_ns(arg['value'], r['NS'])
         if last_err() then return end
-        i = i+1
     end
 end

modules/dev/cname-harvest.lua

@@ -1,5 +1,5 @@
 -- Description: Query for CNAMES to find subdomains
--- Version: 0.2.0
+-- Version: 0.3.0
 -- Source: subdomains
 -- License: GPL-3.0
 
@@ -40,11 +40,9 @@ function run(arg)
     if records['error'] ~= nil then return end
     records = records['answers']
 
-    i = 1
-    while records[i] ~= nil do
+    for i=1, #records do
         r = records[i][2]
         iter(r['CNAME'])
         if last_err() then return end
-        i = i+1
     end
 end

modules/dev/ctlogs.lua

@@ -1,5 +1,5 @@
 -- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.4.0
+-- Version: 0.5.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -70,8 +70,7 @@ function run(arg)
 
     seen = {}
 
-    i = 1
-    while i <= #certs do
+    for i=1, #certs do
         c = certs[i]
         debug(c)
 
@@ -92,15 +91,11 @@ function run(arg)
             if last_err() then return end
             names = crt['valid_names']
 
-            j = 1
-            while j <= #names do
+            for j=1, #names do
                 each_name(names[j])
-                j = j+1
             end
         else
             each_name(c['name_value'])
         end
-
-        i = i+1
     end
 end

modules/dev/cve-2014-8244.lua

@@ -0,0 +1,89 @@
+-- Description: Get connected devices from linksys router via JNAP
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+    local network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    local router = getopt('router')
+    if not router then
+        return 'router option is missing (http://192.0.2.1:9000/)'
+    end
+
+    local target = url_join(router, 'JNAP/')
+    debug(target)
+
+    local session = http_mksession()
+    local headers = {}
+    headers['Content-Type'] = 'application/json; charset=UTF-8'
+    headers['X-JNAP-Action'] = 'http://linksys.com/jnap/core/Transaction'
+    headers['X-JNAP-Authorization'] = 'null'
+    local req = http_request(session, 'POST', target, {
+        headers=headers,
+        body='[{"action":"http://linksys.com/jnap/devicelist/GetDevices","request":{"sinceRevision":0}},{"action":"http://linksys.com/jnap/networkconnections/GetNetworkConnections","request":{}}]',
+    })
+    local r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then
+        return 'http error: ' .. r['status']
+    end
+    -- debug(r)
+
+    body = json_decode(r['text'])
+    if last_err() then return end
+
+    if body['result'] ~= 'OK' then
+        return 'JNAP request failed'
+    end
+
+    local now = datetime()
+    for i=1,#body['responses'] do
+        rr = body['responses'][i]['output']
+        if rr['devices'] then
+            for j=1,#rr['devices'] do
+                device = rr['devices'][j]
+                debug(device)
+
+                name=device['friendlyName']
+                macs=device['knownMACAddresses']
+                connections=device['connections']
+
+                for k=1,#macs do
+                    -- this is an historic entry, do not bump last_seen
+                    device_id = db_add('device', {
+                        value=macs[k],
+                        hostname=name,
+                    })
+                    if last_err() then return end
+                end
+
+                for k=1,#connections do
+                    ipaddr = connections[k]['ipAddress']
+                    mac = connections[k]['macAddress']
+
+                    -- this is an active connection
+                    device_id = db_add('device', {
+                        value=mac,
+                        last_seen=now,
+                    })
+                    if last_err() then return end
+
+                    db_add_ttl('network-device', {
+                        network_id=network_id,
+                        device_id=device_id,
+                        ipaddr=ipaddr,
+                        last_seen=now,
+                    }, 180)
+                    if last_err() then return end
+                end
+            end
+        end
+    end
+end

modules/dev/ddwrt-dhcp.lua

@@ -0,0 +1,88 @@
+-- Description: Export dhcp leases from ddwrt webinterface
+-- Version: 0.2.0
+-- License: GPL-3.0
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    skip_redacted = not getopt('use-redacted')
+
+    router = getopt('router') -- http://192.0.2.1/
+    if not router then
+        return 'router option is missing (http://192.0.2.1/)'
+    end
+    username = getopt('user')
+    password = getopt('password')
+
+    options = {}
+    if username and password then
+        options['basic_auth'] = {username, password}
+    end
+
+    -- request status page
+    session = http_mksession()
+    url = url_join(router, '/Info.live.htm')
+    req = http_request(session, 'GET', url, options)
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then
+        return 'http error: ' .. resp['status']
+    end
+
+    txt = resp['text']
+    debug(txt)
+
+    -- extract leases from response
+    dhcp_section = regex_find('\\{dhcp_leases:: ([^\\}]+)\\}', txt)
+    if last_err() then return end
+    if not dhcp_section then
+        return 'Failed to get dhcp lease section'
+    end
+
+    leases = regex_find_all('\'([^\']+)\',\'([^\']+)\',\'([^\']+)\',\'[^\']+\',\'[^\']+\'', dhcp_section[2])
+    if last_err() then return end
+
+    now = datetime()
+
+    -- add devices to database
+    for i=1, #leases do
+        local hostname = leases[i][2]
+        local ipaddr = leases[i][3]
+        local macaddr = leases[i][4]
+
+        debug({
+            hostname=hostname,
+            ipaddr=ipaddr,
+            macaddr=macaddr,
+        })
+
+        if skip_redacted and macaddr:match('^xx:xx:') then
+            info('Skipping redacted macaddr')
+        else
+            local device = {
+                value=macaddr,
+                last_seen=now,
+            }
+            if hostname ~= '*' then
+                device['hostname'] = hostname
+            end
+
+            local device_id = db_add('device', device)
+
+            db_add_ttl('network-device', {
+                network_id=network_id,
+                device_id=device_id,
+                ipaddr=ipaddr,
+                last_seen=now,
+            }, 120)
+        end
+    end
+end

modules/dev/dns-ns.lua

@@ -0,0 +1,51 @@
+-- Description: Add a domains NS records to scope
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function strip_root_dot(name)
+    local m = regex_find("(.+)\\.$", name)
+    if last_err() then return end
+
+    if m == nil then
+        return name
+    else
+        return m[2]
+    end
+end
+
+function each(r)
+    local name = strip_root_dot(r)
+    local domain = psl_domain_from_dns_name(name)
+    if last_err() then return end
+
+    -- add domain
+    local domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    -- add subdomain
+    local subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+    if last_err() then return end
+end
+
+function run(arg)
+    local records = dns(arg['value'], {
+        record='NS',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    for i=1, #records do
+        local r = records[i][2]
+        debug(r)
+        each(r['NS'])
+        if last_err() then return end
+    end
+end

modules/dev/dns-ptr.lua

@@ -1,5 +1,5 @@
 -- Description: Run reverse dns lookups
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: ipaddrs
 -- License: GPL-3.0
 
@@ -17,8 +17,7 @@ function run(arg)
         if records['error'] ~= nil then return end
         records = records['answers']
 
-        i = 1
-        while records[i] ~= nil do
+        for i=1, #records do
             r = records[i][2]
             if r['PTR'] then
                 db_update('ipaddr', arg, {
@@ -26,7 +25,6 @@ function run(arg)
                 })
                 if last_err() then return end
             end
-            i = i+1
         end
     end
 end

modules/dev/dns-resolve.lua

@@ -1,5 +1,5 @@
 -- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.2.0
+-- Version: 0.3.0
 -- Source: subdomains
 -- License: GPL-3.0
 
@@ -22,8 +22,7 @@ function run(arg)
 
     records = records['answers']
 
-    i = 1
-    while records[i] ~= nil do
+    for i=1, #records do
         r = records[i][2]
         if r['A'] ~= nil then
             ipaddr_id = db_add('ipaddr', {
@@ -38,6 +37,5 @@ function run(arg)
             })
             if last_err() then return end
         end
-        i = i+1
     end
 end

modules/dev/exif.lua

@@ -0,0 +1,12 @@
+-- Description: Extract exif data from images
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    exif = img_exif(arg['value'])
+    if last_err() then return end
+    debug(exif)
+
+    db_update('image', arg, exif)
+end

modules/dev/github.lua

@@ -0,0 +1,107 @@
+-- Description: Collect data from github profiles
+-- Version: 0.2.0
+-- Source: accounts:github.com
+-- License: GPL-3.0
+
+function api_get(url)
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+    -- TODO: set_error(?)
+    if resp['status'] == 403 then return 'ratelimit exceeded' end
+    if resp['status'] ~= 200 then return 'invalid status code' end
+
+    local data = json_decode(resp['text'])
+    if last_err() then return end
+    return data
+end
+
+function import_gpg(url)
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+
+    local key = pgp_pubkey_armored(resp['text'])
+    if not key['uids'] then return end
+
+    for i=1, #key['uids'] do
+        local k = key['uids'][i]
+        debug(k)
+        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", k)
+        if m then
+            db_add('email', {
+                value=m[3],
+                displayname=m[2],
+            })
+        end
+    end
+end
+
+function scan4email(username)
+    local url = 'https://api.github.com/users/' .. username .. '/repos'
+    local repos = api_get(url)
+    if last_err() then return end
+
+    -- XXX: 'https://api.github.com/users/' .. username .. '/events/public?page=0&per_page=100' is faster but less accurate
+
+    for i=1, #repos do
+        local repo = repos[i]
+        debug(repo)
+        local commits = api_get(repo['url'] .. '/commits')
+        if last_err() then return end
+
+        for j=1, #commits do
+            local commit = commits[j]
+            debug(commit)
+
+            if commit['author'] and commit['author']['login'] == username then
+                local name = commit['commit']['author']['name']
+                local email = commit['commit']['author']['email']
+                db_add('email', {
+                    value=email,
+                    displayname=name,
+                })
+                return email
+            end
+
+            if commit['committer'] and commit['committer']['login'] == username then
+                local name = commit['commit']['committer']['name']
+                local email = commit['commit']['committer']['email']
+                db_add('email', {
+                    value=email,
+                    displayname=name,
+                })
+                return email
+            end
+        end
+    end
+end
+
+function run(arg)
+    session = http_mksession()
+    local url = 'https://api.github.com/users/' .. arg['username']
+
+    local data = api_get(url)
+    if last_err() then return end
+    debug(data)
+
+    -- company = data['company']
+    -- location = data['location']
+    -- homepage = data['blog']
+
+    url = 'https://github.com/' .. arg['username'] .. '.gpg'
+    import_gpg(url)
+    if last_err() then return end
+
+    local email = data['email']
+    if not email and not arg['email'] then
+        email = scan4email(arg['username'])
+        if last_err() then return end
+    end
+
+    db_update('account', arg, {
+        url=data['html_url'],
+        displayname=data['name'],
+        email=email,
+    })
+end

modules/dev/hackertarget-subdomains.lua

@@ -1,5 +1,5 @@
 -- Description: Query hackertarget for subdomains of a domain
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -18,13 +18,10 @@ function run(arg)
 
     m = regex_find_all("([^,]+),.+\\n?", resp['text'])
 
-    i = 1
-    while i <= #m do
+    for i=1, #m do
         db_add('subdomain', {
             domain_id=arg['id'],
             value=m[i][2]
         })
-
-        i = i+1
     end
 end

modules/dev/home-assistant-devices.lua

@@ -0,0 +1,56 @@
+-- Description: Query device location from home assistant
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Keyring-Access: home-assistant
+
+function run()
+    -- parsing options
+    instance = getopt('instance')
+    if not instance then
+        return 'instance option is missing'
+    end
+
+    host = url_parse(instance)
+    if last_err() then return end
+    host = host['host']
+
+    entity = getopt('entity')
+    if not entity then
+        return 'entity option is missing'
+    end
+
+    -- fetching credentials
+    creds = keyring('home-assistant:' .. host)
+    if creds[1] == nil then
+        profile = url_join(instance, 'profile')
+        return 'missing home-assistant:' .. host .. ' Long-Lived Access Token, open ' .. profile
+    end
+    token = creds[1]['secret_key']
+
+    headers = {}
+    headers['Authorization'] = 'Bearer ' .. token
+    headers['Content-Type'] = 'application/json'
+
+    -- requesting status
+    session = http_mksession()
+    url = url_join(instance, 'api/states/' .. entity)
+    req = http_request(session, 'GET', url, {
+        headers=headers
+    })
+    r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then
+        return 'http error: ' .. r['status']
+    end
+
+    m = json_decode(r['text'])
+    if last_err() then return end
+    debug(m)
+
+    info({
+        gps_accuracy=m['attributes']['gps_accuracy'],
+        longitude=m['attributes']['longitude'],
+        latitude=m['attributes']['latitude'],
+        last_updated=m['last_updated'],
+    })
+end

modules/dev/images.lua

@@ -0,0 +1,9 @@
+-- Description: Parse image metadata
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    local img = img_load(arg['value'])
+    db_update('image', arg, img)
+end

modules/dev/instagram.lua

@@ -0,0 +1,150 @@
+-- Description: Collect data from instagram profiles
+-- Version: 0.2.0
+-- Source: accounts:instagram.com
+-- License: GPL-3.0
+
+PAGE_SIZE = 50
+
+function get_shared_data(html)
+    local s = html_select_list(html, 'script')
+
+    for i=1, #s do
+        local m = regex_find('^window\\._sharedData = (.+);$', s[i]['text'])
+        if m then
+            return json_decode(m[2])
+        end
+    end
+end
+
+function sign_request(rhx, json_params)
+    local magic = rhx .. ':' .. json_params
+    local x_instagram_gis = hex(md5(magic))
+    return x_instagram_gis
+end
+
+function download_image(node)
+    local url = node['display_url']
+    debug(url)
+
+    local req = http_request(session, 'GET', url, {
+        into_blob=true,
+    })
+    local r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
+
+    db_add('image', {
+        value=r['blob'],
+    })
+end
+
+function pull_graphql(page)
+    local end_cursor = page['page_info']['end_cursor']
+
+    for i=1, #page['edges'] do
+        -- shortcode = page['edges'][i]['shortcode']
+        local node = page['edges'][i]['node']
+        node['thumbnail_resources'] = nil
+        node['media_preview'] = nil
+        -- debug(node)
+
+        -- if node['__typename'] == 'GraphImage'
+
+        -- node['dimensions']['height']
+        -- node['dimensions']['width']
+        -- ^ not sure how to get that picture
+
+        -- node['taken_at_timestamp']
+        -- location = node['location']
+
+        local err = download_image(node)
+        if last_err() then return end
+        if err ~= nil then return err end
+
+        todo_posts = todo_posts -1
+        debug('posts left: ' .. todo_posts .. '/' .. total_posts)
+    end
+
+    if page['page_info']['has_next_page'] then
+        debug('requesting next page=' .. end_cursor)
+
+        variables = json_encode({
+            id=user['id'],
+            first=PAGE_SIZE,
+            after=end_cursor
+        })
+
+        local headers = {}
+        headers['X-Instagram-GIS'] = sign_request(rhx_gis, variables)
+
+        local req = http_request(session, 'GET', 'https://www.instagram.com/graphql/query/', {
+            query={
+                query_hash='42323d64886122307be10013ad2dcc44',
+                variables=variables,
+            },
+            headers=headers,
+        })
+        r = http_send(req)
+        if last_err() then return end
+        if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
+
+        x = json_decode(r['text'])
+        if last_err() then return end
+        return pull_graphql(x['data']['user']['edge_owner_to_timeline_media'])
+    end
+end
+
+function run(arg)
+    session = http_mksession()
+    local url = 'https://www.instagram.com/' .. arg['username'] .. '/'
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'invalid status code' end
+    local html = resp['text']
+
+    local ld = html_select(html, 'script[type="application/ld+json"]')
+    if last_err() then return end
+
+    local ld = json_decode(ld['text'])
+    if last_err() then return end
+    --debug(ld)
+
+    if ld['email'] then
+        db_add('email', {
+            value=ld['email'],
+        })
+    end
+
+    -- homepage=ld['url']
+
+    db_update('account', arg, {
+        displayname=ld['name'],
+        email=ld['email'],
+        url=url,
+    })
+
+    -- download images
+    local sd = get_shared_data(html)
+    if last_err() then return end
+    -- debug(sd)
+
+    rhx_gis = sd['rhx_gis']
+    user = sd['entry_data']['ProfilePage'][1]['graphql']['user']
+
+    -- user['full_name']
+    -- user['id']
+    -- user['is_business_account']
+    -- user['is_private']
+    -- user['is_verified']
+    -- user['has_blocked_viewer']
+    -- user['connected_fb_page']
+    -- user['country_block']
+
+    local page = user['edge_owner_to_timeline_media']
+    total_posts = page['count']
+    todo_posts = total_posts
+
+    -- TODO: fast-update abort if image has been downloaded already
+    return pull_graphql(page)
+end

modules/dev/isc-dhcpd-leases.lua

@@ -1,5 +1,5 @@
 -- Description: Parse isc-dhcpd dhcpd.leases(5)
--- Version: 0.1.0
+-- Version: 0.2.0
 -- License: GPL-3.0
 
 -- cat /var/lib/dhcpd/dhcpd.leases

modules/dev/iw-station-dump.lua

@@ -1,5 +1,5 @@
 -- Description: Parse iw station dump
--- Version: 0.1.0
+-- Version: 0.2.0
 -- License: GPL-3.0
 
 -- iw dev wlan0 station dump

modules/dev/keybase-domains.lua

@@ -0,0 +1,28 @@
+-- Description: Find keybase proofs for domains
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
+        query={
+            domain=arg['value'],
+        }
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    x = json_decode(resp['text'])
+    if last_err() then return end
+    debug(x)
+
+    if x['them'][1] == nil then return end
+    them = x['them'][1]
+
+    db_add('account', {
+        service='keybase.io',
+        username=them['basics']['username'],
+    })
+end

modules/dev/keybase-profiles.lua

@@ -0,0 +1,44 @@
+-- Description: Find keybase proofs for online accounts
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: accounts
+
+function run(arg)
+    service = arg['service']
+    if service == 'twitter.com' then
+        service = 'twitter'
+    elseif service == 'github.com' then
+        service = 'github'
+    elseif service == 'reddit.com' then
+        service = 'reddit'
+    elseif service == 'news.ycombinator.com' then
+        service = 'hackernews'
+    elseif service == 'facebook.com' then
+        service = 'facebook'
+    else
+        return
+    end
+
+    query = {}
+    query[service] = arg['username']
+
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
+        query=query,
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    x = json_decode(resp['text'])
+    if last_err() then return end
+    debug(x)
+
+    if x['them'][1] == nil then return end
+    them = x['them'][1]
+
+    db_add('account', {
+        service='keybase.io',
+        username=them['basics']['username'],
+    })
+end

modules/dev/keybase.lua

@@ -0,0 +1,85 @@
+-- Description: Collect accounts and emails from keybase accounts
+-- Version: 0.2.0
+-- License: GPL-3.0
+-- Source: accounts:keybase.io
+
+function extract_mails(pubkey)
+    for j=1, #pubkey['uids'] do
+        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
+        if m then
+            db_add('email', {
+                value=m[3],
+                displayname=m[2],
+            })
+        end
+    end
+end
+
+function add_domain(dns)
+    local domain = psl_domain_from_dns_name(dns)
+    if last_err() then return end
+
+    local domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    if domain ~= dns then
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=dns,
+        })
+    end
+end
+
+function run(arg)
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
+        query={
+            usernames=arg['username'],
+        }
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    x = json_decode(resp['text'])
+    if last_err() then return end
+    debug(x)
+
+    if x['them'][1] == nil then return end
+    them = x['them'][1]
+
+    -- update keybase profile
+    db_update('account', arg, {
+        displayname=them['profile']['full_name'],
+        url='https://keybase.io/'..arg['username'],
+    })
+
+    -- collect emails
+    pubkey = pgp_pubkey_armored(them['public_keys']['primary']['bundle'])
+    debug(pubkey)
+    extract_mails(pubkey)
+
+    -- collect profiles
+    profiles = them['proofs_summary']['all']
+
+    for i=1, #profiles do
+        profile = profiles[i]
+        debug(profile)
+
+        if
+            profile['proof_type'] == 'generic_web_site' or
+            profile['proof_type'] == 'dns'
+        then
+            add_domain(profile['nametag'])
+        else
+            db_add('account', {
+                service=profile['proof_type'],
+                username=profile['nametag'],
+                url=profile['service_url'],
+            })
+        end
+    end
+end

modules/dev/namechk.lua

@@ -0,0 +1,85 @@
+-- Description: Find accounts by username with namechk.com
+-- Version: 0.2.0
+-- Source: accounts
+-- License: GPL-3.0
+
+function get_services(html)
+    local divs = html_select_list(html, '.service')
+    if last_err() then return end
+
+    local services = {}
+
+    for i=1, #divs do
+        services[i] = divs[i]['attrs']['data-name']
+    end
+
+    return services
+end
+
+function run(arg)
+    -- setup session
+    local session = http_mksession()
+    local req = http_request(session, 'GET', 'https://namechk.com/', {})
+    local resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    local token = html_select(resp['text'], 'input[name="authenticity_token"]')
+    local auth_token = token['attrs']['value']
+
+        local headers = {}
+        headers['X-CSRF-Token'] = authenticity_token
+
+    local services = get_services(resp['text'])
+    debug({
+        auth_token=auth_token,
+        services=services,
+    })
+
+    -- trigger the scan
+    local req = http_request(session, 'POST', 'https://namechk.com/', {
+        headers=headers,
+        form={
+            authenticity_token=auth_token,
+            q=arg['username'],
+        }
+    })
+    local resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+    debug(resp)
+
+    local scan = json_decode(resp['text'])
+    if last_err() then return end
+    local scan_token = scan['valid']
+
+    -- get results
+    for i=1, #services do
+        debug(services[i])
+
+        local req = http_request(session, 'POST', 'https://namechk.com/services/check', {
+            headers=headers,
+            form={
+                token=scan_token,
+                fat=auth_token,
+                service=services[i],
+            }
+        })
+        local resp = http_send(req)
+        if last_err() then return end
+
+        if resp['status'] == 200 then
+            local acc = json_decode(resp['text'])
+            if last_err() then return end
+            debug(acc)
+
+            if acc ~= nil and not acc['available'] and acc['status'] == 'unavailable' then
+                db_add('account', {
+                    service=services[i],
+                    username=arg['username'],
+                    url=acc['callback_url'],
+                })
+            end
+        end
+    end
+end

modules/dev/nudity.lua

@@ -0,0 +1,12 @@
+-- Description: Scan collected images for nudity
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    local nudity = img_nudity(arg['value'])
+    debug(nudity)
+    db_update('image', arg, {
+        nudity=nudity['score'],
+    })
+end

modules/dev/otx-subdomains.lua

@@ -1,5 +1,5 @@
 -- Description: Query alienvault otx passive dns for subdomains of a domain
--- Version: 0.2.0
+-- Version: 0.3.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -18,15 +18,12 @@ function run(arg)
     if last_err() then return end
     o = o['passive_dns']
 
-    i = 1
-    while o[i] do
+    for i=1, #o do
         x = o[i]
 
         db_add('subdomain', {
             domain_id=arg['id'],
             value=x['hostname'],
         })
-
-        i = i+1
     end
 end

modules/dev/passive-spider.lua

@@ -1,5 +1,5 @@
 -- Description: Scrape known http responses for urls
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: urls
 -- License: GPL-3.0
 
@@ -52,12 +52,9 @@ function run(arg)
     end
 
     -- process html links
-    i = 1
-    while i <= #links do
+    for i=1, #links do
         href = links[i]['attrs']['href']
 
         entry(arg['value'], href)
-
-        i = i+1
     end
 end

modules/dev/pgp-keyserver.lua

@@ -1,58 +1,75 @@
 -- Description: Query pgp keyserver for email addresses
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 
+-- TODO: rename to pgp-pks-keyserver
+-- TODO: create pgp-vks-keyserver (?)
+
 function run(arg)
-    session = http_mksession()
+    local domain = arg['value']
 
     --lookup_url = 'https://pgp.mit.edu/pks/lookup'
-    lookup_url = 'https://sks-keyservers.net/pks/lookup'
+    local lookup_url = 'https://sks-keyservers.net/pks/lookup'
 
-    req = http_request(session, 'GET', lookup_url, {
+    session = http_mksession()
+    local req = http_request(session, 'GET', lookup_url, {
         query={
-            search=arg['value'],
+            search=domain,
         }
     })
 
-    resp = http_send(req)
+    local resp = http_send(req)
     if last_err() then return end
     if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
 
     links = html_select_list(resp['text'], 'a')
-    i = 1
-    while i <= #links do
+    debug('keys found: ' .. #links)
+    for i=1, #links do
         href = links[i]['attrs']['href']
 
         if href:find('/pks/lookup%?op=get&search=') == 1 then
             url = url_join(lookup_url, href)
 
+            debug(url)
             req = http_request(session, 'GET', url, {})
-
             resp = http_send(req)
             -- TODO: do not abort script if one attempt fails
             if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
 
-            pubkey = pgp_pubkey_armored(resp['text'])
+            if resp['status'] ~= 200 then
+                error('http error: ' .. url .. ' => ' .. resp['status'])
+            else
+                pubkey = pgp_pubkey_armored(resp['text'])
+                -- print(pubkey)
 
-            -- print(pubkey)
+                emails = {}
+                domain_matched = false
 
-            -- TODO: ensure at least one email matches our target domain
-            if pubkey['uids'] then
-                j = 1
-                while j <= #pubkey['uids'] do
-                    m = regex_find("<([^< ]+@[^< ]+)>$", pubkey['uids'][j])
-                    if m then
-                        db_add('email', {
-                            value=m[2],
-                        })
+                -- TODO: ensure at least one email matches our target domain
+                if pubkey['uids'] then
+                    for j=1, #pubkey['uids'] do
+                        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
+                        if m then
+                            local email = m[3]:lower()
+                            emails[#emails+1] = {
+                                value=email,
+                                displayname=m[2],
+                            }
+
+                            if email:match('@' .. domain .. '$') then
+                                domain_matched = true
+                            end
+                        end
+                    end
+                end
+
+                if domain_matched then
+                    for j=1, #emails do
+                        db_add('email', emails[j])
                     end
-                    j = j+1
                 end
             end
         end
-
-        i = i+1
     end
 end

modules/dev/phpmyadmin.lua

@@ -1,5 +1,5 @@
 -- Description: Search for phpmyadmin
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: urls
 -- License: GPL-3.0
 
@@ -76,8 +76,7 @@ function run(arg)
 
     session = http_mksession()
 
-    i = 1
-    while i <= #paths do
+    for i=1, #paths do
         p = paths[i]
         url = url_join(arg['value'], p)
         debug(url)
@@ -100,7 +99,5 @@ function run(arg)
                 })
             end
         end
-
-        i = i+1
     end
 end

modules/dev/smtp-check.lua

@@ -1,5 +1,5 @@
 -- Description: Verify email address by asking the smtp server
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: emails
 -- License: GPL-3.0
 
@@ -14,14 +14,12 @@ function find_mx(domain)
     records = records['answers']
     -- debug(records)
 
-    i = 1
-    while i <= #records do
+    for i=1, #records do
         r = records[i][2]['MX']
         if r then
             debug('mx: ' .. r[2])
             return r[2]
         end
-        i = i+1
     end
 end
 

modules/dev/threatminer-ipaddr.lua

@@ -1,5 +1,5 @@
 -- Description: Query ThreatMiner passive dns for subdomains of an ip address
--- Version: 0.2.0
+-- Version: 0.3.0
 -- Source: ipaddrs
 -- License: GPL-3.0
 
@@ -23,8 +23,7 @@ function run(arg)
     if last_err() then return end
     o = o['results']
 
-    i = 1
-    while o[i] do
+    for i=1, #o do
         x = o[i]
 
         domain = psl_domain_from_dns_name(x['domain'])
@@ -46,7 +45,5 @@ function run(arg)
                 ip_addr_id=arg['id'],
             })
         end
-
-        i = i+1
     end
 end

modules/dev/threatminer-subdomains.lua

@@ -1,5 +1,5 @@
 -- Description: Query ThreatMiner passive dns for subdomains of a domain
--- Version: 0.2.0
+-- Version: 0.3.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -21,15 +21,12 @@ function run(arg)
     if last_err() then return end
     o = o['results']
 
-    i = 1
-    while o[i] do
+    for i=1, #o do
         x = o[i]
 
         db_add('subdomain', {
             domain_id=arg['id'],
             value=x,
         })
-
-        i = i+1
     end
 end

modules/dev/thunderbird-autoconfig.lua

@@ -1,5 +1,5 @@
 -- Description: Query thunderbird autoconfig db for subdomains
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -27,8 +27,7 @@ function run(arg)
 
     m = regex_find_all('<hostname>([^<]+)</hostname>', resp['text'])
 
-    i = 1
-    while i <= #m do
+    for i=1, #m do
         subdomain = m[i][2]
 
         domain = psl_domain_from_dns_name(subdomain)
@@ -42,7 +41,5 @@ function run(arg)
             value=subdomain,
         })
         if last_err() then return end
-
-        i = i+1
     end
 end

modules/dev/tld-scan.lua

@@ -0,0 +1,46 @@
+-- Description: Search for the same domain base on all TLDs
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    local m = regex_find('^([^\\.]+)\\.', arg['value'])
+    local base = m[2]
+
+    -- TODO: we need a way to cache this
+    -- TODO: .co.uk is missing
+    local url = 'https://data.iana.org/TLD/tlds-alpha-by-domain.txt'
+
+    local session = http_mksession()
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then
+        return 'http error: ' .. resp['status']
+    end
+
+    local tlds = regex_find_all('([^\n]+)', resp['text'])
+
+    for i=1, #tlds do
+        local tld = tlds[i][1]:lower()
+
+        if not tld:match('^#') then
+            local domain = base .. '.' .. tld
+
+            debug(domain)
+            records = dns(domain, {
+                record='NS',
+            })
+            if last_err() then
+                clear_err()
+            else
+                if records['error'] == nil and records['answers'][1] then
+                    debug(records)
+                    db_add('domain', {
+                        value=domain,
+                    })
+                end
+            end
+        end
+    end
+end

modules/dev/url-scan.lua

@@ -1,5 +1,5 @@
 -- Description: Scan subdomains for websites
--- Version: 0.2.0
+-- Version: 0.3.0
 -- Source: subdomains
 -- License: GPL-3.0
 

modules/dev/viewdns-reverseip.lua

@@ -0,0 +1,29 @@
+-- Description: Search for domains pointing to ip
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: ipaddrs
+
+-- XXX: the api ignores subdomains
+function run(arg)
+    local session = http_mksession()
+    local req = http_request(session, 'GET', 'https://viewdns.info/reverseip/', {
+        query={
+            host=arg['value'],
+            t='1',
+        }
+    })
+    local r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then
+        return 'http error: ' .. r['status']
+    end
+    local html = r['text']
+
+    local domains = html_select_list(html, '#null table td:first-child')
+    for i=2, #domains do
+        local domain = domains[i]['text']
+        db_add('domain', {
+            value=domain,
+        })
+    end
+end

modules/dev/waybackurls.lua

@@ -1,5 +1,5 @@
 -- Description: Discover subdomains from wayback machine
--- Version: 0.3.0
+-- Version: 0.4.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -29,8 +29,7 @@ function run(arg)
 
     seen = {}
 
-    i = 2
-    while o[i] do
+    for i=2, #o do
         url = o[i][3]
         debug(url)
         parts = url_parse(url)
@@ -51,7 +50,5 @@ function run(arg)
                 seen[subdomain] = 1
             end
         end
-
-        i = i+1
     end
 end

modules/dev/well-known-uris.lua

@@ -1,5 +1,5 @@
 -- Description: Scan for known /.well-known/ locations
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: urls
 -- License: GPL-3.0
 
@@ -24,8 +24,7 @@ function run(arg)
 
     session = http_mksession()
 
-    i = 1
-    while i <= #locations do
+    for i=1, #locations do
         path = locations[i]['path']
         expect_redirect = locations[i]['redirect']
 
@@ -58,7 +57,5 @@ function run(arg)
                 db_add('url', obj)
             end
         end
-
-        i = i+1
     end
 end

modules/harness/port.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    ip_addr = '192.168.1.2'
+    ip_addr_id = db_add('ipaddr', {
+        value=ip_addr,
+    })
+    db_add('port', {
+        ip_addr_id=ip_addr_id,
+        ip_addr=ip_addr,
+        protocol='tcp',
+        port=4444,
+        status='open',
+    })
+end

modules/harness/tux-img.lua

@@ -0,0 +1,21 @@
+-- Description: Download an image
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+
+    req = http_request(session, 'GET', 'https://www.kernel.org/theme/images/logos/tux.png', {
+        into_blob=true,
+    })
+    r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then
+        return 'http error: ' .. r['status']
+    end
+
+    debug(r)
+    db_add('image', {
+        value=r['blob'],
+    })
+end

modules/harness/warn.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run()
+    warn('testing')
+    warn_once('testing once')
+    warn_once('testing once')
+end

modules/harness/xss.lua

@@ -0,0 +1,8 @@
+-- Description: jaVasCript:/*-/*`/*\`/*'/*"/**/(/* */oNcliCk=alert() )//%0D%0A%0d%0a//</stYle/</titLe/</teXtarEa/</scRipt/--!>\x3csVg/<sVg/oNloAd=alert()//>\x3e
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run()
+    -- TODO your code here
+end

sn0int-registry/Cargo.toml

@@ -8,22 +8,22 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.4.0", path="sn0int-common" }
-rocket = "0.4"
-rocket_failure = { version = "0.1", features = ["with-rocket"] }
-rocket_contrib = { version = "0.4", features = ["handlebars_templates"] }
+sn0int-common = { version="0.7.0", path="sn0int-common" }
+rocket = { version = "0.4", default-features=false }
+rocket_failure = { version = "0.1.2", features = ["with-rocket"] }
+rocket_contrib = { version = "0.4.1", features = ["handlebars_templates"] }
 
 diesel = { version = "1.3", features = ["postgres", "r2d2"] }
 diesel_migrations = { version = "1.3.0", features = ["postgres"] }
 diesel_full_text_search = "1.0.1"
 
 reqwest = "0.9.2"
-oauth2 = "2.0.0-alpha.2"
+oauth2 = "2.0.0-beta.2"
 failure = "0.1"
 url = "1.0"
 log = "0.4"
 semver = "0.9.0"
-lazy_static = "1.1"
+lazy_static = "1"
 blake2 = "0.8.0"
 hex = "0.3.1"
 maplit = "1.0.1"

sn0int-registry/assets/style.css

@@ -2,8 +2,10 @@ body {
     background-color: black;
     color: #00aa00;
     font-family: monospace;
-    max-width: 700px;
-    margin: 0 auto;
+    max-width: 900px;
+    margin: 20px auto;
+    font-size: 14px;
+    line-height: 18px;
 }
 
 input {
@@ -29,7 +31,41 @@ a {
     padding: 10px;
 }
 
-.list-unstyled {
-    list-style: none;
-    padding: 0 0 0 20px;
+.center {
+    text-align: center;
+}
+
+.modules {
+    list-style: none;
+    padding: 0;
+    word-break: break-word;
+}
+
+.modules li {
+    padding: 2px 5px;
+    margin: 1px 0;
+    background-color: #222;
+}
+
+.mod-cell {
+    display: inline-block;
+    vertical-align: top;
+}
+
+.mod-name {
+    width: 35%;
+}
+
+.mod-name a {
+    text-decoration: none;
+    color: #00cc00;
+    font-weight: bold;
+}
+
+.mod-version {
+    width: 15%;
+}
+
+.mod-descr {
+    width: 50%;
 }

sn0int-registry/migrations/2019-03-02-130236_publish-time/down.sql

@@ -0,0 +1 @@
+ALTER TABLE releases DROP COLUMN published;

sn0int-registry/migrations/2019-03-02-130236_publish-time/up.sql

@@ -0,0 +1 @@
+ALTER TABLE releases ADD COLUMN published TIMESTAMP NOT NULL DEFAULT NOW();

sn0int-registry/migrations/2019-05-27-000932_source/down.sql

@@ -0,0 +1,19 @@
+ALTER TABLE modules DROP COLUMN source;
+
+CREATE OR REPLACE FUNCTION modules_vector_update() RETURNS TRIGGER AS $$
+BEGIN
+    IF TG_OP = 'INSERT' THEN
+        new.search_vector = to_tsvector('pg_catalog.english',
+            NEW.name || ' ' || NEW.author || ' ' || NEW.description
+        );
+    END IF;
+    IF TG_OP = 'UPDATE' THEN
+        IF NEW.description <> OLD.description THEN
+            new.search_vector = to_tsvector('pg_catalog.english',
+                NEW.name || ' ' || NEW.author || ' ' || NEW.description
+            );
+        END IF;
+    END IF;
+    RETURN NEW;
+END
+$$ LANGUAGE 'plpgsql';

sn0int-registry/migrations/2019-05-27-000932_source/up.sql

@@ -0,0 +1,19 @@
+ALTER TABLE modules ADD COLUMN source VARCHAR;
+
+CREATE OR REPLACE FUNCTION modules_vector_update() RETURNS TRIGGER AS $$
+BEGIN
+    IF TG_OP = 'INSERT' THEN
+        new.search_vector = to_tsvector('pg_catalog.english',
+            NEW.name || ' ' || NEW.author || ' ' || COALESCE(NEW.source, '') || ' ' || NEW.description
+        );
+    END IF;
+    IF TG_OP = 'UPDATE' THEN
+        IF NEW.description <> OLD.description OR NEW.source <> OLD.source THEN
+            new.search_vector = to_tsvector('pg_catalog.english',
+                NEW.name || ' ' || NEW.author || ' ' || COALESCE(NEW.source, '') || ' ' || NEW.description
+            );
+        END IF;
+    END IF;
+    RETURN NEW;
+END
+$$ LANGUAGE 'plpgsql';

sn0int-registry/sn0int-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.4.0"
+version = "0.7.0"
 description = "Common code for sn0int"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"

sn0int-registry/sn0int-common/src/api.rs

@@ -46,3 +46,8 @@ impl SearchResponse {
         format!("{}/{}", self.author, self.name)
     }
 }
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct LatestResponse {
+    pub time: Option<u64>,
+}

sn0int-registry/sn0int-common/src/metadata.rs

@@ -35,9 +35,35 @@ pub enum Source {
     Urls,
     Emails,
     PhoneNumbers,
+    Networks,
+    Devices,
+    Accounts(Option<String>),
+    Breaches,
+    Images,
+    Ports,
     KeyRing(String),
 }
 
+impl Source {
+    pub fn group_as_str(&self) -> &'static str {
+        match self {
+            Source::Domains => "domains",
+            Source::Subdomains => "subdomains",
+            Source::IpAddrs => "ipaddrs",
+            Source::Urls => "urls",
+            Source::Emails => "emails",
+            Source::PhoneNumbers => "phonenumbers",
+            Source::Networks => "networks",
+            Source::Devices => "devices",
+            Source::Accounts(_) => "accounts",
+            Source::Breaches => "breaches",
+            Source::Images => "images",
+            Source::Ports => "ports",
+            Source::KeyRing(_) => "keyring",
+        }
+    }
+}
+
 impl FromStr for Source {
     type Err = Error;
 
@@ -56,6 +82,12 @@ impl FromStr for Source {
             ("urls", None) => Ok(Source::Urls),
             ("emails", None) => Ok(Source::Emails),
             ("phonenumbers", None) => Ok(Source::PhoneNumbers),
+            ("networks", None) => Ok(Source::Networks),
+            ("devices", None) => Ok(Source::Devices),
+            ("accounts", param) => Ok(Source::Accounts(param.map(String::from))),
+            ("breaches", None) => Ok(Source::Breaches),
+            ("images", None) => Ok(Source::Images),
+            ("ports", None) => Ok(Source::Ports),
             ("keyring", Some(param)) => Ok(Source::KeyRing(param.to_string())),
             (x, Some(param)) => bail!("Unknown Source: {:?} ({:?})", x, param),
             (x, None) => bail!("Unknown Source: {:?}", x),
@@ -244,4 +276,13 @@ mod tests {
         let x = Source::from_str("keyring");
         assert!(x.is_err());
     }
+
+    #[test]
+    fn verify_account_source() {
+        let x = Source::from_str("accounts").unwrap();
+        assert_eq!(x, Source::Accounts(None));
+
+        let x = Source::from_str("accounts:github.com").unwrap();
+        assert_eq!(x, Source::Accounts(Some("github.com".into())));
+    }
 }

sn0int-registry/src/auth.rs

@@ -2,7 +2,7 @@ use crate::errors::*;
 use diesel::pg::PgConnection;
 use oauth2::basic::BasicClient;
 use oauth2::prelude::*;
-use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl};
+use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl, TokenResponse};
 use crate::github::GithubAuthenticator;
 use crate::models::AuthToken;
 use url::Url;

sn0int-registry/src/main.rs

@@ -78,12 +78,17 @@ fn run() -> Result<()> {
         }))
         .mount("/api/v0", routes![
             routes::api::quickstart,
+            routes::api::latest,
             routes::api::search,
             routes::api::info,
             routes::api::download,
             routes::api::publish,
             routes::api::whoami,
         ])
+        .mount("/api/v0/badges", routes![
+            routes::badges::modules,
+            routes::badges::downloads,
+        ])
         .mount("/auth", routes![
             routes::auth::get,
             routes::auth::post,
@@ -93,6 +98,7 @@ fn run() -> Result<()> {
             routes::assets::index,
             routes::assets::favicon,
             routes::assets::style,
+            routes::health::health,
         ])
         .register(catchers![
             bad_request,

sn0int-registry/src/models.rs

@@ -4,6 +4,9 @@ use diesel::pg::PgConnection;
 use diesel::sql_types::BigInt;
 use diesel_full_text_search::{plainto_tsquery, TsQueryExtensions};
 use crate::schema::*;
+use std::collections::HashMap;
+use std::time::SystemTime;
+use sn0int_common::metadata::Metadata;
 
 
 #[derive(AsChangeset, Serialize, Deserialize, Queryable, Insertable)]
@@ -50,6 +53,7 @@ type AllModuleColumns = (
     modules::description,
     modules::latest,
     modules::featured,
+    modules::source,
 );
 
 pub const ALL_MODULE_COLUMNS: AllModuleColumns = (
@@ -59,6 +63,7 @@ pub const ALL_MODULE_COLUMNS: AllModuleColumns = (
     modules::description,
     modules::latest,
     modules::featured,
+    modules::source,
 );
 
 #[derive(AsChangeset, Identifiable, Queryable, Serialize, PartialEq, Debug)]
@@ -70,6 +75,7 @@ pub struct Module {
     pub description: String,
     pub latest: Option<String>,
     pub featured: bool,
+    pub source: Option<String>,
 }
 
 impl Module {
@@ -98,7 +104,10 @@ impl Module {
                         .map_err(Error::from)
     }
 
-    pub fn update_or_create(author: &str, name: &str, description: &str, connection: &PgConnection) -> Result<Module> {
+    pub fn update_or_create(author: &str, name: &str, metadata: &Metadata, connection: &PgConnection) -> Result<Module> {
+        let description = metadata.description.as_str();
+        let source = metadata.source.as_ref().map(|x| x.group_as_str());
+
         match Self::find_opt(author, name, connection)? {
             Some(module) => diesel::update(modules::table.filter(modules::columns::id.eq(module.id)))
                             .set(modules::columns::description.eq(description))
@@ -110,6 +119,7 @@ impl Module {
                 name,
                 description,
                 latest: None,
+                source,
             }, connection),
         }
     }
@@ -152,14 +162,15 @@ impl Module {
     pub fn search(query: &str, connection: &PgConnection) -> Result<Vec<(Module, i64)>> {
         let q = plainto_tsquery(query);
 
-        let x: Vec<(i32, String, String, String, Option<String>, bool, i64)> = modules::table.select((
+        let x: Vec<(i32, String, String, String, Option<String>, bool, Option<String>, i64)> = modules::table.select((
                 modules::id,
                 modules::author,
                 modules::name,
                 modules::description,
                 modules::latest,
                 modules::featured,
-                diesel::dsl::sql::<BigInt>("sum(releases.downloads) AS sum"),
+                modules::source,
+                diesel::dsl::sql::<BigInt>("coalesce(sum(releases.downloads), 0) AS sum"),
             ))
             .left_join(releases::table)
             .group_by(modules::id)
@@ -170,7 +181,7 @@ impl Module {
             ))
             .load(connection)?;
 
-        Ok(x.into_iter().map(|(id, author, name, description, latest, featured, downloads)| (
+        Ok(x.into_iter().map(|(id, author, name, description, latest, featured, source, downloads)| (
             Module {
                 id,
                 author,
@@ -178,6 +189,7 @@ impl Module {
                 description,
                 latest,
                 featured,
+                source,
             },
             downloads,
         )).collect())
@@ -194,6 +206,60 @@ impl Module {
             .load(connection)
             .map_err(Error::from)
     }
+
+    pub fn start_page(connection: &PgConnection) -> Result<HashMap<String, Vec<Module>>> {
+        let x: Vec<(i32, String, String, String, Option<String>, bool, Option<String>, i64)> = modules::table.select((
+                modules::id,
+                modules::author,
+                modules::name,
+                modules::description,
+                modules::latest,
+                modules::featured,
+                modules::source,
+                diesel::dsl::sql::<BigInt>("coalesce(sum(releases.downloads), 0) AS sum"),
+            ))
+            .left_join(releases::table)
+            .group_by(modules::id)
+            //.filter(q.matches(modules::search_vector))
+            .filter(modules::source.is_not_null())
+            .order((
+                modules::featured.desc(),
+                diesel::dsl::sql::<BigInt>("sum").desc(),
+            ))
+            .load(connection)?;
+
+        let mut categories: HashMap<_, Vec<_>> = HashMap::new();
+
+        for (id, author, name, description, latest, featured, source, _downloads) in x {
+            let module = Module {
+                id,
+                author,
+                name,
+                description,
+                latest,
+                featured,
+                source,
+            };
+            if let Some(source) = &module.source {
+                if let Some(cat) = categories.get_mut(source) {
+                    cat.push(module);
+                } else {
+                    categories.insert(source.clone(), vec![module]);
+                }
+            }
+        }
+
+        Ok(categories)
+    }
+
+    pub fn count(connection: &PgConnection) -> Result<i64> {
+        use diesel::dsl::*;
+
+        modules::table
+            .select(count(modules::id))
+            .first(connection)
+            .map_err(Error::from)
+    }
 }
 
 #[derive(Insertable)]
@@ -203,6 +269,7 @@ pub struct NewModule<'a> {
     name: &'a str,
     description: &'a str,
     latest: Option<&'a str>,
+    source: Option<&'a str>,
 }
 
 #[derive(AsChangeset, Identifiable, Queryable, Associations, Serialize, PartialEq, Debug)]
@@ -214,6 +281,7 @@ pub struct Release {
     pub version: String,
     pub downloads: i32,
     pub code: String,
+    pub published: SystemTime,
 }
 
 impl Release {
@@ -271,6 +339,21 @@ impl Release {
             .execute(connection)?;
         Ok(())
     }
+
+    pub fn latest(connection: &PgConnection) -> Result<Option<Release>> {
+        releases::table
+            .order_by(releases::published.desc())
+            .first::<Release>(connection)
+            .optional()
+            .map_err(Error::from)
+    }
+
+    pub fn downloads(connection: &PgConnection) -> Result<i64> {
+        releases::table
+            .select(diesel::dsl::sql::<BigInt>("coalesce(sum(releases.downloads), 0) AS sum"))
+            .first(connection)
+            .map_err(Error::from)
+    }
 }
 
 #[derive(Insertable)]

sn0int-registry/src/routes/api.rs

@@ -9,6 +9,7 @@ use semver::Version;
 use sn0int_common::api::*;
 use sn0int_common::id;
 use sn0int_common::metadata::Metadata;
+use std::time::UNIX_EPOCH;
 
 
 #[get("/quickstart")]
@@ -17,6 +18,20 @@ pub fn quickstart(connection: db::Connection) -> ApiResult<ApiResponse<Vec<Modul
     Ok(ApiResponse::Success(modules))
 }
 
+#[get("/latest")]
+pub fn latest(connection: db::Connection) -> ApiResult<ApiResponse<LatestResponse>> {
+    let time = Release::latest(&connection)?
+        .map(|x| {
+            x.published.duration_since(UNIX_EPOCH)
+                .expect("Time went backwards")
+                .as_secs()
+        });
+    let latest = LatestResponse {
+        time,
+    };
+    Ok(ApiResponse::Success(latest))
+}
+
 #[derive(Debug, FromForm)]
 pub struct Search {
     q: String,
@@ -103,7 +118,7 @@ pub fn publish(name: String, upload: Json<PublishRequest>, session: AuthHeader,
         .public_context("Version is invalid")?;
 
     connection.transaction::<_, WebError, _>(|| {
-        let module = Module::update_or_create(&user, &name, &metadata.description, &connection)
+        let module = Module::update_or_create(&user, &name, &metadata, &connection)
             .private_context("Failed to write module metadata")?;
 
         match Release::try_find(module.id, &version, &connection)? {

sn0int-registry/src/routes/assets.rs

@@ -1,15 +1,20 @@
 use crate::assets::{ASSET_REV, FAVICON, STYLE_SHEET};
-use rocket::http::ContentType;
-use rocket::http::Status;
+use crate::db;
+use crate::errors::ApiResult;
+use crate::models::*;
+use rocket::http::{ContentType, Status};
 use rocket::http::hyper::header::{CacheControl, CacheDirective};
 use rocket_contrib::templates::Template;
 
 
 #[get("/")]
-pub fn index() -> Template {
-    Template::render("index", hashmap!{
-        "ASSET_REV" => ASSET_REV.as_str(),
-    })
+pub fn index(connection: db::Connection) -> ApiResult<Template> {
+    let modules = Module::start_page(&connection)?;
+    let asset_rev = ASSET_REV.as_str();
+    Ok(Template::render("index", json!({
+        "ASSET_REV": asset_rev,
+        "modules": modules,
+    })))
 }
 
 #[derive(Responder)]

sn0int-registry/src/routes/badges.rs

@@ -0,0 +1,45 @@
+use crate::errors::*;
+use crate::db;
+use crate::models::*;
+use rocket_contrib::json::Json;
+
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Badge {
+    schema_version: u8,
+    label: String,
+    message: String,
+    color: &'static str,
+}
+
+impl Badge {
+    pub fn new(label: String, message: String) -> Badge {
+        Badge {
+            schema_version: 1,
+            label,
+            message,
+            color: "blue",
+        }
+    }
+}
+
+#[get("/modules")]
+pub fn modules(connection: db::Connection) -> ApiResult<Json<Badge>> {
+    let num = Module::count(&connection)?;
+    Ok(Json(Badge::new("modules".into(), num.to_string())))
+}
+
+#[get("/downloads")]
+pub fn downloads(connection: db::Connection) -> ApiResult<Json<Badge>> {
+    let mut num = Release::downloads(&connection)?;
+
+    let mut unit = String::new();
+    while num > 1000 {
+        num /= 1000;
+        unit.push('k');
+    }
+
+    let message = format!("{:.1}{}", num, unit);
+    Ok(Json(Badge::new("downloads".into(), message)))
+}

sn0int-registry/src/routes/health.rs

@@ -0,0 +1,8 @@
+use crate::errors::*;
+use crate::db;
+
+
+#[get("/health")]
+pub fn health(_connection: db::Connection) -> ApiResult<()> {
+    Ok(())
+}

sn0int-registry/src/routes/mod.rs

@@ -1,3 +1,5 @@
 pub mod api;
 pub mod assets;
 pub mod auth;
+pub mod badges;
+pub mod health;

sn0int-registry/src/schema.rs

@@ -21,6 +21,7 @@ table! {
         latest -> Nullable<Varchar>,
         search_vector -> Tsvector,
         featured -> Bool,
+        source -> Nullable<Varchar>,
     }
 }
 
@@ -34,6 +35,7 @@ table! {
         version -> Varchar,
         downloads -> Int4,
         code -> Text,
+        published -> Timestamp,
     }
 }
 

sn0int-registry/templates/base.html.hbs

@@ -6,7 +6,16 @@
         <link rel="stylesheet" href="/assets/{{ASSET_REV}}/style.css">
     </head>
     <body>
-        <h1><a href="/">sn0int</a></h1>
+        <h1 class="center"><a href="/">sn0int registry</a></h1>
+        <nav>
+            <p class="center">
+                [ <a href="https://github.com/kpcyrd/sn0int" target="_blank" rel="noopener">github</a> ]
+                [ <a href="https://sn0int.readthedocs.io/" target="_blank" rel="noopener">docs</a> ]
+                [ <a href="https://www.reddit.com/r/sn0int" target="_blank" rel="noopener">reddit</a> ]
+                [ <a href="https://chaos.social/@sn0int" target="_blank" rel="noopener">mastodon</a> ]
+                [ <a href="https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int" target="_blank" rel="noopener">irc</a> ]
+            </p>
+        </nav>
 
         {{~> page}}
     </body>

sn0int-registry/templates/index.html.hbs

@@ -1,35 +1,10 @@
 {{#*inline "page"}}
         <p>
-This is the registry server of sn0int, a semi-automatic OSINT framework and
-package manager.  It was built for IT security professionals and bug hunters to
-gather intelligence about a given target or about yourself. sn0int is
-enumerating attack surface by semi-automatically processing public information
-and mapping the results in a unified format for followup investigations.
-        </p>
-
-        <p>
-Among other things, sn0int is currently able to:
-        </p>
-
-        <ul class="list-unstyled">
-<li>- Harvest subdomains from certificate transparency logs</li>
-<li>- Harvest subdomains from various passive dns logs</li>
-<li>- Sift through subdomain results for publicly accessible websites</li>
-<li>- Harvest emails from pgp keyservers</li>
-<li>- Enrich ip addresses with ASN and geoip info</li>
-<li>- Harvest subdomains from the wayback machine</li>
-<li>- Gather information about phonenumbers</li>
-<li>- Bruteforce interesting urls</li>
-        </ul>
-
-        <p>
-sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
-and is fully opensource.  None of the investigations listed above are hardcoded
-in the source, instead those are provided by modules that are executed in a
-sandbox. You can easily extend sn0int by writing your own modules and share
-them with other users by publishing them to the sn0int registry. This allows
-you to ship updates for your modules on your own since you don't need to send a
-pull request.
+        This is the registry server of sn0int, a semi-automatic OSINT framework and
+        package manager.  It was built for IT security professionals and bug hunters to
+        gather intelligence about a given target or about yourself. sn0int is
+        enumerating attack surface by semi-automatically processing public information
+        and mapping the results in a unified format for followup investigations.
         </p>
 
         <p>
@@ -40,37 +15,21 @@ pull request.
         </p>
 
         <p>
-        You can search for modules with sn0int using:
-        </p>
-        <p class="code"><code>
-        sn0int search ctlogs
-        </code></p>
-
-        <p>
-        To install the module afterwards run:
+        To install a module run:
         </p>
         <p class="code"><code>
         sn0int install kpcyrd/ctlogs
         </code></p>
 
-        <p>
-        If you want to publish your own module, you need to login with github first:
-        </p>
-        <p class="code"><code>
-        sn0int login
-        </code></p>
-
-        <p>
-        Afterwards you can upload your module to the registry (this is going to
-        publish your module as yourgithubuser/example):
-        </p>
-        <p class="code"><code>
-        sn0int publish ./path/to/example.lua
-        </code></p>
-
-        <p>
-        For more information, visit <a href="https://github.com/kpcyrd/sn0int"
-        target="_blank" rel="noopener">sn0int on github</a>.
-        </p>
+        {{#each modules}}
+            <h3>[ {{@key}} ]</h3>
+            <ul class="modules">
+            {{#each this}}
+                <li>
+                    <span class="mod-cell mod-name"><a href="#">{{this.author}}/{{this.name}}</a></span><span class="mod-cell mod-version">{{this.latest}}</span><span class="mod-cell mod-descr">{{this.description}}</span>
+                </li>
+            {{/each}}
+            </ul>
+        {{/each}}
 {{/inline}}
 {{~> base }}

src/api.rs

@@ -46,6 +46,7 @@ impl Client {
             info!("Adding session token to request");
             request.header("Auth", session.as_str());
         }
+        request.header("User-Agent", web::default_user_agent());
 
         let request = request.body(body)?;
 
@@ -131,4 +132,10 @@ impl Client {
         let reply = self.get::<Vec<ModuleInfoResponse>>(&url)?;
         Ok(reply)
     }
+
+    pub fn latest_release(&self) -> Result<LatestResponse> {
+        let url = format!("{}/api/v0/latest", self.server);
+        let reply = self.get::<LatestResponse>(&url)?;
+        Ok(reply)
+    }
 }

src/args.rs

@@ -9,8 +9,8 @@ use crate::workspaces::Workspace;
 #[structopt(author = "",
             raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
-    #[structopt(short="w", long="workspace")]
     /// Select a different workspace instead of the default
+    #[structopt(short="w", long="workspace")]
     pub workspace: Option<Workspace>,
 
     #[structopt(subcommand)]
@@ -28,66 +28,78 @@ impl Args {
 
 #[derive(Debug, StructOpt)]
 pub enum SubCommand {
-    #[structopt(author="", name="run")]
     /// Run a module directly
+    #[structopt(author="", name="run")]
     Run(Run),
-    #[structopt(author="", name="sandbox")]
     /// For internal use
+    #[structopt(author="", name="sandbox")]
     Sandbox(Sandbox),
-    #[structopt(author="", name="login")]
     /// Login to the registry for publishing
+    #[structopt(author="", name="login")]
     Login(Login),
-    #[structopt(author="", name="new")]
     /// Create a new module
+    #[structopt(author="", name="new")]
     New(New),
-    #[structopt(author="", name="publish")]
     /// Publish a script to the registry
+    #[structopt(author="", name="publish")]
     Publish(Publish),
-    #[structopt(author="", name="install")]
     /// Install a module from the registry
+    #[structopt(author="", name="install")]
     Install(Install),
-    #[structopt(author="", name="search")]
     /// Search in the registry
+    #[structopt(author="", name="search")]
     Search(Search),
-    #[structopt(author="", name="select")]
+    /// Insert into the database
+    #[structopt(author="", name="add")]
+    Add(cmd::add_cmd::Args),
     /// Select from the database
+    #[structopt(author="", name="select")]
     Select(cmd::select_cmd::Args),
-    #[structopt(author="", name="completions")]
+    /// Delete from the database
+    #[structopt(author="", name="delete")]
+    Delete(cmd::delete_cmd::Args),
+    /// Manage workspaces
+    #[structopt(author="", name="workspace")]
+    Workspace(cmd::workspace_cmd::Args),
+    /// Verify blob storage for corrupt and dangling blobs
+    #[structopt(author="", name="fsck")]
+    Fsck(cmd::fsck_cmd::Args),
     /// Generate shell completions
+    #[structopt(author="", name="completions")]
     Completions(Completions),
 }
 
 #[derive(Debug, StructOpt)]
 pub struct Run {
     /// Execute a module that has been installed
-    pub module: Option<String>,
-    #[structopt(short="f", long="file", conflicts_with="module")]
+    pub module: String,
     /// Run a module from a path
-    pub file: Option<String>,
-    #[structopt(short="j", long="threads", default_value="1")]
+    #[structopt(short="f", long="file")]
+    pub file: bool,
     /// Run modules concurrently
+    #[structopt(short="j", long="threads", default_value="1")]
     pub threads: usize,
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
     /// Verbose logging, once to print inserts even if they don't add new
     /// data, twice to activate the debug() function
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
     pub verbose: u64,
-    #[structopt(long="stdin")]
     /// Expose stdin to modules
+    #[structopt(long="stdin")]
     pub stdin: bool,
-    #[structopt(long="grant")]
     /// Automatically grant access to a keyring namespace
+    #[structopt(long="grant")]
     pub grants: Vec<String>,
-    #[structopt(long="grant-full-keyring")]
     /// Automatically grant access to all requested keys
+    #[structopt(long="grant-full-keyring")]
     pub grant_full_keyring: bool,
-    #[structopt(long="deny-keyring")]
     /// Automatically deny access to all requested keys
+    #[structopt(long="deny-keyring")]
     pub deny_keyring: bool,
-    #[structopt(short="x", long="exit-on-error")]
     /// Exit on first error and set exit code
+    #[structopt(short="x", long="exit-on-error")]
     pub exit_on_error: bool,
-    #[structopt(short="o", long="option")]
     /// Set an option
+    #[structopt(short="o", long="option")]
     pub options: Vec<options::Opt>,
 }
 

src/auth.rs

@@ -1,6 +1,7 @@
 use crate::errors::*;
 use opener;
 use std::fs;
+use std::path::PathBuf;
 use std::thread;
 use std::time::Duration;
 use crate::api::Client;
@@ -9,17 +10,18 @@ use crate::paths;
 use crate::term;
 
 
-pub fn load_token() -> Result<String> {
+fn path() -> Result<PathBuf> {
     let path = paths::data_dir()?;
-    let path = path.join("auth");
-    let session = fs::read_to_string(path)?;
+    Ok(path.join("auth"))
+}
+
+pub fn load_token() -> Result<String> {
+    let session = fs::read_to_string(path()?)?;
     Ok(session.trim().to_string())
 }
 
 pub fn save_token(session: &str) -> Result<()> {
-    let path = paths::data_dir()?;
-    let path = path.join("auth");
-    fs::write(path, format!("{}\n", session))?;
+    fs::write(path()?, format!("{}\n", session))?;
     Ok(())
 }
 

src/blobs.rs

@@ -0,0 +1,241 @@
+use crate::errors::*;
+use crate::paths;
+use crate::worker::{EventWithCallback, Event2};
+use crate::workspaces::Workspace;
+
+use blake2::VarBlake2b;
+use digest::{Input, VariableOutput};
+use bytes::Bytes;
+use serde::ser::{Serialize, Serializer};
+use serde::de::{self, Deserialize, Deserializer};
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::result;
+use std::sync::mpsc;
+
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Blob {
+    pub id: String,
+    pub bytes: Bytes,
+}
+
+impl Blob {
+    pub fn create(bytes: Bytes) -> Blob {
+        let id = Self::hash(&bytes);
+        Blob {
+            id,
+            bytes,
+        }
+    }
+
+    pub fn hash(bytes: &[u8]) -> String {
+        let mut h = VarBlake2b::new(32).unwrap();
+        h.input(bytes);
+        Self::encode_hash(&h.vec_result())
+    }
+
+    #[inline]
+    fn encode_hash(bytes: &[u8]) -> String {
+        let x = bs58::encode(bytes).into_string();
+        format!("{:0<44}", x)
+    }
+}
+
+impl EventWithCallback for Blob {
+    type Payload = ();
+
+    #[inline(always)]
+    fn with_callback(self, tx: mpsc::Sender<result::Result<Self::Payload, String>>) -> Event2 {
+        Event2::Blob((self, tx))
+    }
+}
+
+impl Serialize for Blob {
+    #[inline]
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let s = base64::encode(&self.bytes);
+        serializer.serialize_str(&s)
+    }
+}
+
+impl<'de> Deserialize<'de> for Blob {
+    #[inline]
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let bytes = base64::decode(&s)
+            .map_err(de::Error::custom)?;
+        Ok(Blob::create(Bytes::from(bytes)))
+    }
+}
+
+pub struct BlobStorage {
+    path: PathBuf,
+}
+
+impl BlobStorage {
+    #[inline]
+    pub fn new<I: Into<PathBuf>>(path: I) -> BlobStorage {
+        BlobStorage {
+            path: path.into(),
+        }
+    }
+
+    #[inline]
+    pub fn workspace(workspace: &Workspace) -> Result<BlobStorage> {
+        let path = paths::blobs_dir(workspace)?;
+        Ok(BlobStorage::new(path))
+    }
+
+    #[inline]
+    pub fn join(&self, id: &str) -> Result<PathBuf> {
+        if !id.chars().all(char::is_alphanumeric) {
+            bail!("blob id contains invalid characters");
+        }
+        Ok(self.path.join(id))
+    }
+
+    #[inline(always)]
+    pub fn path(&self) -> &Path {
+        self.path.as_ref()
+    }
+
+    pub fn load(&self, id: &str) -> Result<Blob> {
+        let path = self.join(id)?;
+
+        debug!("Loading blob from {:?}", path);
+        let bytes = fs::read(path)
+            .context("Failed to read blob")?;
+
+        Ok(Blob {
+            id: id.to_string(),
+            bytes: Bytes::from(bytes),
+        })
+    }
+
+    pub fn save(&self, blob: &Blob) -> Result<()> {
+        let path = self.join(&blob.id)?;
+
+        debug!("Writing blob to {:?}", path);
+        fs::write(path, &blob.bytes)
+            .context("Failed to write blob")?;
+
+        Ok(())
+    }
+
+    pub fn delete(&self, id: &str) -> Result<()> {
+        let path = self.join(id)?;
+        debug!("Deleting blob: {:?}", path);
+        fs::remove_file(path)
+            .context("Failed to delete blob")?;
+        Ok(())
+    }
+
+    pub fn list(&self) -> Result<Vec<String>> {
+        let mut blobs = Vec::new();
+        for entry in fs::read_dir(&self.path)? {
+            let blob = entry?
+                .file_name()
+                .into_string()
+                .map_err(|_| format_err!("Invalid filename"))?;
+            blobs.push(blob);
+        }
+        Ok(blobs)
+    }
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile;
+    use serde_json;
+
+    #[inline]
+    fn blob() -> (Bytes, Blob) {
+        let bytes = Bytes::from(&b"asdf"[..]);
+        (bytes.clone(), Blob::create(bytes))
+    }
+
+    #[test]
+    fn verify_create_blob() {
+        let (bytes, blob) = blob();
+        assert_eq!(blob, Blob {
+            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
+            bytes,
+        });
+    }
+
+    #[test]
+    fn test_blobstorage_save() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let s = BlobStorage::new(dir.path());
+
+        let (_, blob1) = blob();
+        s.save(&blob1).expect("save failed");
+
+        let blob2 = s.load("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz").expect("load failed");
+
+        assert_eq!(blob1, blob2);
+    }
+
+    #[test]
+    fn test_blobstorage_load_failure() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let s = BlobStorage::new(dir.path());
+
+        let result = s.load("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz");
+
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_path_validation() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let s = BlobStorage::new(dir.path());
+
+        let result = s.load("../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../etc/passwd");
+
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_serialize() {
+        let (_, blob) = blob();
+        let json = serde_json::to_string(&blob).unwrap();
+        assert_eq!(&json, "\"YXNkZg==\"");
+    }
+
+    #[test]
+    fn test_deserialize() {
+        let (_, blob1) = blob();
+        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
+        assert_eq!(blob1, blob2);
+    }
+
+    #[test]
+    fn test_hash_encoding() {
+        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
+    }
+
+    #[test]
+    fn test_hash_encoding_padding() {
+        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
+    }
+}

src/cmd/add_cmd.rs

@@ -1,28 +1,97 @@
 use crate::errors::*;
 
+use crate::blobs::Blob;
+use crate::cmd::Cmd;
+use crate::gfx;
 use crate::models::*;
 use crate::shell::Readline;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::utils;
+use crate::term;
+use std::fs;
+use std::net;
+use std::path::Path;
+use walkdir::WalkDir;
 
 
 #[derive(Debug, StructOpt)]
 #[structopt(author = "",
             raw(global_settings = "&[AppSettings::ColoredHelp]"))]
-pub enum Args {
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Target,
+    /// Do not actually insert into database
+    #[structopt(short="n", long="dry-run")]
+    dry_run: bool,
+}
+
+#[derive(Debug, StructOpt)]
+pub enum Target {
+    /// Insert domain into the database
     #[structopt(name="domain")]
     Domain(AddDomain),
+    /// Insert subdomain into the database
     #[structopt(name="subdomain")]
     Subdomain(AddSubdomain),
+    /// Insert ip address into the database
+    #[structopt(name="ipaddr")]
+    IpAddr(AddIpAddr),
+    /// Insert email into the database
     #[structopt(name="email")]
     Email(AddEmail),
+    /// Insert phonenumber into the database
     #[structopt(name="phonenumber")]
     PhoneNumber(AddPhoneNumber),
+    /// Insert device into the database
     #[structopt(name="device")]
     Device(AddDevice),
+    /// Insert network into the database
     #[structopt(name="network")]
     Network(AddNetwork),
+    /// Insert account into the database
+    #[structopt(name="account")]
+    Account(AddAccount),
+    /// Insert breach into the database
+    #[structopt(name="breach")]
+    Breach(AddBreach),
+    /// Insert images into the database
+    #[structopt(name="image")]
+    Image(AddImage),
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Readline) -> Result<()> {
+        match self.subcommand {
+            Target::Domain(args) => args.insert(rl, self.dry_run),
+            Target::Subdomain(args) => args.insert(rl, self.dry_run),
+            Target::IpAddr(args) => args.insert(rl, self.dry_run),
+            Target::Email(args) => args.insert(rl, self.dry_run),
+            Target::PhoneNumber(args) => args.insert(rl, self.dry_run),
+            Target::Device(args) => args.insert(rl, self.dry_run),
+            Target::Network(args) => args.insert(rl, self.dry_run),
+            Target::Account(args) => args.insert(rl, self.dry_run),
+            Target::Breach(args) => args.insert(rl, self.dry_run),
+            Target::Image(args) => args.insert(rl, self.dry_run),
+        }
+    }
+}
+
+#[inline]
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    Args::run_str(rl, args)
+}
+
+trait IntoInsert: Sized {
+    fn into_insert(self, rl: &Readline) -> Result<Insert>;
+
+    fn insert(self, rl: &Readline, dry_run: bool) -> Result<()> {
+        let insert = self.into_insert(rl)?;
+        if !dry_run {
+            rl.db().insert_generic(insert)?;
+        }
+        Ok(())
+    }
 }
 
 #[derive(Debug, StructOpt)]
@@ -30,28 +99,181 @@ pub struct AddDomain {
     domain: Option<String>,
 }
 
+impl IntoInsert for AddDomain {
+    fn into_insert(self, rl: &Readline) -> Result<Insert> {
+        let domain = match self.domain {
+            Some(domain) => domain,
+            _ => utils::question("Domain")?,
+        };
+
+        // ensure input is a valid domain
+        let parsed_domain = rl.psl().parse_domain(&domain)
+            .map_err(|e| format_err!("Failed to parse domain: {}", e))?;
+
+        if Some(domain.as_str()) != parsed_domain.root() {
+            bail!("This is not a valid domain, might be a subdomain or tld");
+        }
+
+        Ok(Insert::Domain(NewDomain {
+            value: domain,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddSubdomain {
     subdomain: Option<String>,
 }
 
+impl IntoInsert for AddSubdomain {
+    fn into_insert(self, rl: &Readline) -> Result<Insert> {
+        let subdomain = match self.subdomain {
+            Some(subdomain) => subdomain,
+            _ => utils::question("Subdomain")?,
+        };
+
+        let dns_name = rl.psl().parse_dns_name(&subdomain)
+            .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
+
+        let domain = dns_name.domain()
+            .ok_or_else(|| format_err!("Dns Name seems invalid"))?
+            .to_string();
+
+        let domain_id = match rl.db().insert_struct(NewDomain {
+            value: domain,
+        })? {
+            Some((_, domain_id)) => domain_id,
+            _ => bail!("Domain is out out of scope"),
+        };
+
+        Ok(Insert::Subdomain(NewSubdomain {
+            domain_id,
+            value: subdomain,
+            resolvable: None,
+        }))
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AddIpAddr {
+    ipaddr: Option<net::IpAddr>,
+}
+
+impl IntoInsert for AddIpAddr {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let ipaddr = match self.ipaddr {
+            Some(ipaddr) => ipaddr,
+            _ => {
+                let ipaddr = utils::question("IP address")?;
+                ipaddr.parse()?
+            },
+        };
+
+        let family = match ipaddr {
+            net::IpAddr::V4(_) => "4",
+            net::IpAddr::V6(_) => "6",
+        };
+
+        Ok(Insert::IpAddr(NewIpAddr {
+            family: family.to_string(),
+            value: ipaddr.to_string(),
+            continent: None,
+            continent_code: None,
+            country: None,
+            country_code: None,
+            city: None,
+            latitude: None,
+            longitude: None,
+            asn: None,
+            as_org: None,
+            description: None,
+            reverse_dns: None,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddEmail {
     email: Option<String>,
 }
 
+impl IntoInsert for AddEmail {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let email = match self.email {
+            Some(email) => email,
+            _ => utils::question("Email")?,
+        };
+
+        Ok(Insert::Email(NewEmail {
+            value: email,
+            displayname: None,
+            valid: None,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddPhoneNumber {
     phonenumber: Option<String>,
     name: Option<String>,
 }
 
+impl IntoInsert for AddPhoneNumber {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let (phonenumber, name) = match self.phonenumber {
+            Some(phonenumber) => (phonenumber, self.name),
+            _ => {
+                let phonenumber = utils::question("Phone Number")?;
+                let name = utils::question_opt("Name")?;
+                (phonenumber, name)
+            },
+        };
+
+        Ok(Insert::PhoneNumber(NewPhoneNumber {
+            value: phonenumber,
+            name: name,
+            valid: None,
+            last_online: None,
+            country: None,
+            carrier: None,
+            line: None,
+            is_ported: None,
+            last_ported: None,
+            caller_name: None,
+            caller_type: None,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddDevice {
     mac: Option<String>,
     name: Option<String>,
 }
 
+impl IntoInsert for AddDevice {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let (mac, name) = match self.mac {
+            Some(mac) => {
+                (mac, self.name)
+            },
+            _ => {
+                let mac = utils::question("Mac address")?;
+                let name = utils::question_opt("Name")?;
+                (mac, name)
+            },
+        };
+
+        Ok(Insert::Device(NewDevice {
+            value: mac,
+            name: name,
+            hostname: None,
+            vendor: None,
+            last_seen: None,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddNetwork {
     network: Option<String>,
@@ -59,150 +281,177 @@ pub struct AddNetwork {
     longitude: Option<f32>,
 }
 
-pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    match args {
-        Args::Domain(args) => add_domain(rl, args),
-        Args::Subdomain(args) => add_subdomain(rl, args),
-        Args::Email(args) => add_email(rl, args),
-        Args::PhoneNumber(args) => add_phonenumber(rl, args),
-        Args::Device(args) => add_device(rl, args),
-        Args::Network(args) => add_network(rl, args),
+impl IntoInsert for AddNetwork {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let (network, latitude, longitude) = match self.network {
+            Some(network) => (network, self.latitude, self.longitude),
+            _ => {
+                let network = utils::question("Network")?;
+                let latitude = utils::question_typed_opt("Latitude")?;
+                let longitude = utils::question_typed_opt("Longitude")?;
+                (network, latitude, longitude)
+            }
+        };
+
+        Ok(Insert::Network(NewNetwork {
+            value: network,
+            latitude,
+            longitude,
+        }))
     }
 }
 
-fn add_domain(rl: &mut Readline, args: AddDomain) -> Result<()> {
-    let domain = match args.domain {
-        Some(domain) => domain,
-        _ => utils::question("Domain")?,
-    };
-
-    // ensure input is a valid domain
-    let parsed_domain = rl.psl().parse_domain(&domain)
-        .map_err(|e| format_err!("Failed to parse domain: {}", e))?;
-
-    if Some(domain.as_str()) != parsed_domain.root() {
-        bail!("This is not a valid domain, might be a subdomain or tld");
-    }
-
-    rl.db().insert_struct(NewDomain {
-        value: &domain,
-    })?;
-
-    Ok(())
+#[derive(Debug, StructOpt)]
+pub struct AddAccount {
+    service: Option<String>,
+    username: Option<String>,
 }
 
-fn add_subdomain(rl: &mut Readline, args: AddSubdomain) -> Result<()> {
-    let subdomain = match args.subdomain {
-        Some(subdomain) => subdomain,
-        _ => utils::question("Subdomain")?,
-    };
+impl IntoInsert for AddAccount {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let (service, username) = match (self.service, self.username) {
+            (Some(service), Some(username)) => (service, username),
+            _ => {
+                let service = utils::question("Service")?;
+                let username = utils::question("Username")?;
 
-    let dns_name = rl.psl().parse_dns_name(&subdomain)
-        .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
+                (service, username)
+            },
+        };
 
-    let domain = dns_name.domain()
-        .ok_or_else(|| format_err!("Dns Name seems invalid"))?
-        .to_string();
-
-    let domain_id = match rl.db().insert_struct(NewDomain {
-        value: &domain,
-    })? {
-        Some((_, domain_id)) => domain_id,
-        _ => bail!("Domain is out out of scope"),
-    };
-
-    rl.db().insert_struct(NewSubdomain {
-        domain_id,
-        value: &subdomain,
-        resolvable: None,
-    })?;
-
-    Ok(())
-}
-
-fn add_email(rl: &mut Readline, args: AddEmail) -> Result<()> {
-    let email = match args.email {
-        Some(email) => email,
-        _ => utils::question("Email")?,
-    };
-
-    rl.db().insert_struct(NewEmail {
-        value: &email,
-        valid: None,
-    })?;
-
-    Ok(())
-}
-
-fn add_phonenumber(rl: &mut Readline, args: AddPhoneNumber) -> Result<()> {
-    let (phonenumber, name) = match args.phonenumber {
-        Some(phonenumber) => {
-            (phonenumber, args.name)
-        },
-        _ => {
-            let phonenumber = utils::question("Phone Number")?;
-            let name = utils::question_opt("Name")?;
-            (phonenumber, name)
-        },
-    };
-
-    rl.db().insert_struct(NewPhoneNumber {
-        value: &phonenumber,
-        name: name.as_ref(),
-        valid: None,
-        last_online: None,
-        country: None,
-        carrier: None,
-        line: None,
-        is_ported: None,
-        last_ported: None,
-        caller_name: None,
-        caller_type: None,
-    })?;
-
-    Ok(())
-}
-
-fn add_device(rl: &mut Readline, args: AddDevice) -> Result<()> {
-    let (mac, name) = match args.mac {
-        Some(mac) => {
-            (mac, args.name)
-        },
-        _ => {
-            let mac = utils::question("Mac address")?;
-            let name = utils::question_opt("Name")?;
-            (mac, name)
-        },
-    };
-
-    rl.db().insert_struct(NewDevice {
-        value: &mac,
-        name: name.as_ref(),
-        hostname: None,
-        vendor: None,
-        last_seen: None,
-    })?;
-
-    Ok(())
-}
-
-fn add_network(rl: &mut Readline, args: AddNetwork) -> Result<()> {
-    let (network, latitude, longitude) = match args.network {
-        Some(network) => (network, args.latitude, args.longitude),
-        _ => {
-            let network = utils::question("Network")?;
-            let latitude = utils::question_typed_opt("Latitude")?;
-            let longitude = utils::question_typed_opt("Longitude")?;
-            (network, latitude, longitude)
+        if service.contains('/') {
+            // TODO: avoid duplication
+            bail!("Service field can't contain `/`");
         }
-    };
 
-    rl.db().insert_struct(NewNetwork {
-        value: &network,
-        latitude,
-        longitude,
-    })?;
+        let value = format!("{}/{}", service, username);
 
-    Ok(())
+        Ok(Insert::Account(NewAccount {
+            value: value,
+            service: service,
+            username: username,
+            displayname: None,
+            email: None,
+            url: None,
+            last_seen: None,
+        }))
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AddBreach {
+    name: Option<String>,
+}
+
+impl IntoInsert for AddBreach {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let name = match self.name {
+            Some(name) => name,
+            _ => {
+                let name = utils::question("Name")?;
+                name
+            }
+        };
+
+        Ok(Insert::Breach(NewBreach {
+            value: name,
+        }))
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AddImage {
+    paths: Vec<String>,
+}
+
+impl IntoInsert for AddImage {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        unreachable!()
+    }
+
+    fn insert(self, rl: &Readline, dry_run: bool) -> Result<()> {
+        let paths = if self.paths.is_empty() {
+            let path = utils::question("Path")?;
+            vec![path]
+        } else {
+            self.paths
+        };
+
+        for path in paths {
+            for path in WalkDir::new(path) {
+                let path = match path {
+                    Ok(path) => path,
+                    Err(err) => {
+                        let path = err.path().unwrap_or(Path::new("")).display();
+
+                        let err = err.io_error()
+                            .map(|err| err.to_string())
+                            .unwrap_or_else(|| String::from("walkdir failed"));
+                        term::error(&format!("Failed to access entry {:?}: {}", path, err));
+
+                        continue;
+                    },
+                };
+
+                if path.file_type().is_dir() {
+                    debug!("Traversing into directory: {:?}", path);
+                    continue;
+                }
+
+                debug!("Testing: {:?}", path.path());
+
+                let data = match fs::read(path.path()) {
+                    Ok(data) => data,
+                    Err(err) => {
+                        term::error(&format!("Failed to read {:?}: {}", path, err));
+                        continue;
+                    },
+                };
+
+                // check if image
+                let format = match gfx::guess_format(&data) {
+                    Ok(format) => format.mime().to_string(),
+                    _ => {
+                        debug!("Probably not an image, skipping");
+                        continue;
+                    },
+                };
+                debug!("Detected image format: {:?}", format);
+
+                let blob = Blob::create(data.into());
+                if !dry_run {
+                    rl.blobs().save(&blob)?;
+                }
+                let value = blob.id;
+
+                let filename = path.file_name()
+                    .to_string_lossy()
+                    .to_string();
+
+                term::info(&format!("{} {:?}", value, path.path()));
+
+                if !dry_run {
+                    rl.db().insert_generic(Insert::Image(NewImage {
+                        value,
+
+                        filename: Some(filename),
+                        mime: Some(format),
+                        width: None,
+                        height: None,
+                        created: None,
+
+                        latitude: None,
+                        longitude: None,
+
+                        nudity: None,
+                        ahash: None,
+                        dhash: None,
+                        phash: None,
+                    }))?;
+                }
+            }
+        }
+
+        Ok(())
+    }
 }

src/cmd/delete_cmd.rs

@@ -1,6 +1,7 @@
 use crate::errors::*;
 
-use crate::db;
+use crate::cmd::Cmd;
+use crate::filters::{Target, Filter};
 use crate::shell::Readline;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
@@ -11,52 +12,38 @@ use crate::term;
 #[derive(Debug, StructOpt)]
 #[structopt(author = "",
             raw(global_settings = "&[AppSettings::ColoredHelp]"))]
-pub enum Args {
-    #[structopt(name="domains")]
-    Domains(Filter),
-    #[structopt(name="subdomains")]
-    Subdomains(Filter),
-    #[structopt(name="ipaddrs")]
-    IpAddrs(Filter),
-    #[structopt(name="urls")]
-    Urls(Filter),
-    #[structopt(name="emails")]
-    Emails(Filter),
-    #[structopt(name="phonenumbers")]
-    PhoneNumbers(Filter),
-    #[structopt(name="devices")]
-    Devices(Filter),
-    #[structopt(name="networks")]
-    Networks(Filter),
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Target,
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Filter {
-    args: Vec<String>,
-}
-
-impl Filter {
-    pub fn parse(&self) -> Result<db::Filter> {
-        db::Filter::parse(&self.args)
+impl Cmd for Args {
+    fn run(self, rl: &mut Readline) -> Result<()> {
+        let rows = match &self.subcommand {
+            Target::Domains(filter) => delete::<Domain>(rl, &filter),
+            Target::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
+            Target::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
+            Target::Urls(filter) => delete::<Url>(rl, &filter),
+            Target::Emails(filter) => delete::<Email>(rl, &filter),
+            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
+            Target::Devices(filter) => delete::<Device>(rl, &filter),
+            Target::Networks(filter) => delete::<Network>(rl, &filter),
+            Target::Accounts(filter) => delete::<Account>(rl, &filter),
+            Target::Breaches(filter) => delete::<Breach>(rl, &filter),
+            Target::Images(filter) => delete::<Image>(rl, &filter),
+            Target::Ports(filter) => delete::<Port>(rl, &filter),
+        }?;
+        term::info(&format!("Deleted {} rows", rows));
+        Ok(())
     }
 }
 
+#[inline]
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    let rows = match args {
-        Args::Domains(filter) => delete::<Domain>(rl, &filter),
-        Args::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
-        Args::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
-        Args::Urls(filter) => delete::<Url>(rl, &filter),
-        Args::Emails(filter) => delete::<Email>(rl, &filter),
-        Args::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
-        Args::Devices(filter) => delete::<Device>(rl, &filter),
-        Args::Networks(filter) => delete::<Network>(rl, &filter),
-    }?;
-    term::info(&format!("Deleted {} rows", rows));
-    Ok(())
+    Args::run_str(rl, args)
 }
 
+#[inline]
 fn delete<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<usize> {
-    rl.db().delete::<T>(&filter.parse()?)
+    T::delete(rl.db(), &filter.parse()?)
 }

src/cmd/fsck_cmd.rs

@@ -0,0 +1,88 @@
+use crate::errors::*;
+
+use crate::blobs::Blob;
+use crate::cmd::Cmd;
+use crate::shell::Readline;
+use crate::term;
+use crate::worker;
+use crate::models::*;
+use std::collections::HashSet;
+use structopt::StructOpt;
+
+
+#[derive(Debug, StructOpt)]
+pub struct Args {
+    /// Verbose output
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
+    verbose: u64,
+    /// Delete only dangling blobs
+    #[structopt(long="gc")]
+    gc: bool,
+    /// Delete dangling and corrupted blobs
+    #[structopt(long="gc-all")]
+    gc_all: bool,
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Readline) -> Result<()> {
+        let blobs = rl.blobs();
+
+        let hashset = worker::spawn_fn("Building reference set...", || {
+            let mut hashset = HashSet::new();
+
+            for x in rl.db().list::<Image>()? {
+                hashset.insert(x.value);
+            }
+
+            Ok(hashset)
+        }, true)?;
+
+        for blob in blobs.list()? {
+
+            let state = worker::spawn_fn(&blob, || {
+                let blob = rl.blobs().load(&blob)?;
+
+                // ensure content matches hash
+                if Blob::hash(&blob.bytes) != blob.id {
+                    Ok(State::Corrupted)
+                // ensure blob is referenced by any table
+                } else if !hashset.contains(&blob.id) {
+                    Ok(State::Dangling)
+                } else {
+                    Ok(State::Valid)
+                }
+            }, true);
+
+            match state {
+                Ok(State::Valid) => {
+                    if self.verbose > 0 {
+                        term::info(&format!("{}... ok", blob));
+                    }
+                },
+                Ok(State::Dangling) => {
+                    term::warn(&format!("{}... dangling", blob));
+                    if self.gc || self.gc_all {
+                        blobs.delete(&blob)?;
+                    }
+                },
+                Ok(State::Corrupted) => {
+                    term::error(&format!("{}... corrupted", blob));
+                    if self.gc_all {
+                        blobs.delete(&blob)?;
+                    }
+                },
+                Err(err) => {
+                    term::error(&format!("{}... {}", blob, err));
+                },
+            }
+        }
+
+        Ok(())
+    }
+}
+
+enum State {
+    Valid,
+    Dangling,
+    Corrupted,
+}

src/cmd/mod.rs

@@ -2,7 +2,7 @@ use crate::errors::*;
 use crate::shell::Readline;
 
 pub trait Cmd: structopt::StructOpt + Sized {
-    fn run(&self, rl: &mut Readline) -> Result<()>;
+    fn run(self, rl: &mut Readline) -> Result<()>;
 
     #[inline]
     fn run_str(rl: &mut Readline, args: &[String]) -> Result<()> {
@@ -13,6 +13,7 @@ pub trait Cmd: structopt::StructOpt + Sized {
 
 pub mod add_cmd;
 pub mod delete_cmd;
+pub mod fsck_cmd;
 pub mod help_cmd;
 pub mod run_cmd;
 pub mod use_cmd;

src/cmd/mod_cmd.rs

@@ -8,6 +8,7 @@ use colored::Colorize;
 use crate::engine::Module;
 use crate::registry;
 use crate::shell::Readline;
+use crate::update::AutoUpdater;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::term;
@@ -24,20 +25,20 @@ pub struct Args {
 
 #[derive(Debug, StructOpt)]
 pub enum SubCommand {
-    #[structopt(author="", name="list")]
     /// List installed modules
+    #[structopt(author="", name="list")]
     List(List),
-    #[structopt(author="", name="install")]
     /// Install module from registry
+    #[structopt(author="", name="install")]
     Install(args::Install),
-    #[structopt(author="", name="search")]
     /// Search modules in registry
+    #[structopt(author="", name="search")]
     Search(args::Search),
-    #[structopt(author="", name="reload")]
     /// Reload modules
+    #[structopt(author="", name="reload")]
     Reload(Reload),
-    #[structopt(author="", name="update")]
     /// Update modules
+    #[structopt(author="", name="update")]
     Update(Update),
 }
 
@@ -54,18 +55,27 @@ pub struct Update {
 }
 
 fn update(client: &Client, config: &Config, module: &Module) -> Result<()> {
+    let name = module.canonical();
     let installed = module.version();
-    let infos = client.query_module(&module.id())?;
+
+    let label = format!("Searching for updates {}", name);
+    let infos = worker::spawn_fn(&label, || {
+        client.query_module(&module.id())
+    }, true)?;
     debug!("Latest version: {:?}", infos);
 
     let latest = infos.latest.ok_or_else(|| format_err!("Module doesn't have any released versions"))?;
 
     if installed != latest {
-        term::info(&format!("Updating {}: {:?} -> {:?}", module.canonical(), installed, latest));
-        registry::run_install(&Install {
-            module: module.id(),
-            version: None,
-        }, &config)?;
+        let label = format!("Updating {}: {:?} -> {:?}", &name, installed, latest);
+        worker::spawn_fn(&label, || {
+            registry::run_install(&Install {
+                module: module.id(),
+                version: None,
+            }, &config)
+        }, true)?;
+
+        term::success(&format!("Updated {}: {:?} -> {:?}", &name, installed, latest));
     }
 
     Ok(())
@@ -105,17 +115,25 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
         SubCommand::Update(_) => {
             let client = Client::new(&config)?;
 
+            let mut success = true;
+
             for module in rl.engine().list() {
-                let name = module.canonical();
-                let label = format!("Searching for updates {}", name);
-
-                let result = worker::spawn_fn(&label, || {
-                    update(&client, &config, &module)
-                }, true);
-
-                if let Err(err) = result {
-                    term::error(&format!("Failed to update {}: {:?}", name, err));
+                if module.is_private() {
+                    debug!("{} is a private module, skipping", module.canonical());
+                    continue;
                 }
+
+                if let Err(err) = update(&client, &config, &module) {
+                    term::error(&format!("Failed to update {}: {}", module.canonical(), err));
+                    success = false;
+                }
+            }
+
+            // TODO: keep a list of outdated packages and remove them after they've been updated
+            if success {
+                let mut autoupdate = AutoUpdater::load()?;
+                autoupdate.all_updated();
+                autoupdate.save()?;
             }
 
             // trigger reload

src/cmd/noscope_cmd.rs

@@ -1,6 +1,6 @@
 use crate::errors::*;
 
-use crate::db;
+use crate::filters::{Target, Filter};
 use crate::shell::Readline;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
@@ -11,46 +11,32 @@ use crate::term;
 #[derive(Debug, StructOpt)]
 #[structopt(author = "",
             raw(global_settings = "&[AppSettings::ColoredHelp]"))]
-pub enum Args {
-    #[structopt(name="domains")]
-    Domains(Filter),
-    #[structopt(name="subdomains")]
-    Subdomains(Filter),
-    #[structopt(name="ipaddrs")]
-    IpAddrs(Filter),
-    #[structopt(name="urls")]
-    Urls(Filter),
-    #[structopt(name="emails")]
-    Emails(Filter),
-    #[structopt(name="phonenumbers")]
-    PhoneNumbers(Filter),
-}
-
-#[derive(Debug, StructOpt)]
-pub struct Filter {
-    args: Vec<String>,
-}
-
-impl Filter {
-    pub fn parse(&self) -> Result<db::Filter> {
-        db::Filter::parse(&self.args)
-    }
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Target,
 }
 
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
-    let rows = match args {
-        Args::Domains(filter) => noscope::<Domain>(rl, &filter),
-        Args::Subdomains(filter) => noscope::<Subdomain>(rl, &filter),
-        Args::IpAddrs(filter) => noscope::<IpAddr>(rl, &filter),
-        Args::Urls(filter) => noscope::<Url>(rl, &filter),
-        Args::Emails(filter) => noscope::<Email>(rl, &filter),
-        Args::PhoneNumbers(filter) => noscope::<PhoneNumber>(rl, &filter),
+    let rows = match args.subcommand {
+        Target::Domains(filter) => noscope::<Domain>(rl, &filter),
+        Target::Subdomains(filter) => noscope::<Subdomain>(rl, &filter),
+        Target::IpAddrs(filter) => noscope::<IpAddr>(rl, &filter),
+        Target::Urls(filter) => noscope::<Url>(rl, &filter),
+        Target::Emails(filter) => noscope::<Email>(rl, &filter),
+        Target::PhoneNumbers(filter) => noscope::<PhoneNumber>(rl, &filter),
+        Target::Devices(filter) => noscope::<Device>(rl, &filter),
+        Target::Networks(filter) => noscope::<Network>(rl, &filter),
+        Target::Accounts(filter) => noscope::<Account>(rl, &filter),
+        Target::Breaches(filter) => noscope::<Breach>(rl, &filter),
+        Target::Images(filter) => noscope::<Image>(rl, &filter),
+        Target::Ports(filter) => noscope::<Port>(rl, &filter),
     }?;
     term::info(&format!("Updated {} rows", rows));
     Ok(())
 }
 
+#[inline]
 fn noscope<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<usize> {
-    rl.db().noscope::<T>(&filter.parse()?)
+    T::noscope(rl.db(), &filter.parse()?)
 }

src/cmd/run_cmd.rs

@@ -1,8 +1,8 @@
 use crate::errors::*;
 
 use crate::args;
-use crate::db::{Database, Filter};
-use crate::db::ttl;
+use crate::blobs::{Blob, BlobStorage};
+use crate::db::{ttl, Filter};
 use crate::engine::Module;
 use crate::models::*;
 use crate::shell::Readline;
@@ -67,16 +67,26 @@ impl From<Args> for Params<'static> {
     }
 }
 
-fn prepare_arg<T: Serialize + Model>(x: T) -> Result<(serde_json::Value, Option<String>)> {
+fn prepare_arg<T: Serialize + Model>(bs: &BlobStorage, x: T) -> Result<(serde_json::Value, Option<String>, Vec<Blob>)> {
     let pretty = x.to_string();
+
+    let blobs = if let Some(blob) = x.blob() {
+        let blob = bs.load(blob)?;
+        vec![blob]
+    } else {
+        Vec::new()
+    };
+
     let arg = serde_json::to_value(x)?;
-    Ok((arg, Some(pretty)))
+    Ok((arg, Some(pretty), blobs))
 }
 
-fn prepare_args<T: Scopable + Serialize + Model>(db: &Database, filter: &Filter) -> Result<Vec<(serde_json::Value, Option<String>)>> {
-    db.filter::<T>(filter)?
+fn prepare_args<T: Scopable + Serialize + Model>(rl: &Readline, filter: &Filter, param: Option<&String>) -> Result<Vec<(serde_json::Value, Option<String>, Vec<Blob>)>> {
+    let db = rl.db();
+    let bs = rl.blobs();
+    db.filter_with_param::<T>(filter, param)?
         .into_iter()
-        .map(prepare_arg)
+        .map(|x| prepare_arg(bs, x))
         .collect()
 }
 
@@ -113,12 +123,18 @@ pub fn execute(rl: &mut Readline, params: Params, options: HashMap<String, Strin
     let filter = rl.scoped_targets();
 
     let args = match module.source() {
-        Some(Source::Domains) => prepare_args::<Domain>(rl.db(), &filter),
-        Some(Source::Subdomains) => prepare_args::<Subdomain>(rl.db(), &filter),
-        Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl.db(), &filter),
-        Some(Source::Urls) => prepare_args::<Url>(rl.db(), &filter),
-        Some(Source::Emails) => prepare_args::<Email>(rl.db(), &filter),
-        Some(Source::PhoneNumbers) => prepare_args::<PhoneNumber>(rl.db(), &filter),
+        Some(Source::Domains) => prepare_args::<Domain>(rl, &filter, None),
+        Some(Source::Subdomains) => prepare_args::<Subdomain>(rl, &filter, None),
+        Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl, &filter, None),
+        Some(Source::Urls) => prepare_args::<Url>(rl, &filter, None),
+        Some(Source::Emails) => prepare_args::<Email>(rl, &filter, None),
+        Some(Source::PhoneNumbers) => prepare_args::<PhoneNumber>(rl, &filter, None),
+        Some(Source::Networks) => prepare_args::<Network>(rl, &filter, None),
+        Some(Source::Devices) => prepare_args::<Device>(rl, &filter, None),
+        Some(Source::Accounts(service)) => prepare_args::<Account>(rl, &filter, service.as_ref()),
+        Some(Source::Breaches) => prepare_args::<Breach>(rl, &filter, None),
+        Some(Source::Images) => prepare_args::<Image>(rl, &filter, None),
+        Some(Source::Ports) => prepare_args::<Port>(rl, &filter, None),
         Some(Source::KeyRing(namespace)) => {
             let keyring = rl.keyring();
             if keyring.is_access_granted(&module, &namespace) {
@@ -126,14 +142,14 @@ pub fn execute(rl: &mut Readline, params: Params, options: HashMap<String, Strin
                     .map(|key| {
                         let pretty = format!("{}:{}", key.namespace, key.access_key);
                         let arg = serde_json::to_value(key)?;
-                        Ok((arg, Some(pretty)))
+                        Ok((arg, Some(pretty), vec![]))
                     })
                     .collect::<Result<Vec<_>>>()
             } else {
                 Ok(vec![])
             }
         },
-        None => Ok(vec![(serde_json::Value::Null, None)]),
+        None => Ok(vec![(serde_json::Value::Null, None, vec![])]),
     }?;
 
     rl.signal_register().catch_ctrl();

src/cmd/scope_cmd.rs

@@ -1,6 +1,6 @@
 use crate::errors::*;
 
-use crate::db;
+use crate::filters::{Target, Filter};
 use crate::shell::Readline;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
@@ -11,46 +11,32 @@ use crate::term;
 #[derive(Debug, StructOpt)]
 #[structopt(author = "",
             raw(global_settings = "&[AppSettings::ColoredHelp]"))]
-pub enum Args {
-    #[structopt(name="domains")]
-    Domains(Filter),
-    #[structopt(name="subdomains")]
-    Subdomains(Filter),
-    #[structopt(name="ipaddrs")]
-    IpAddrs(Filter),
-    #[structopt(name="urls")]
-    Urls(Filter),
-    #[structopt(name="emails")]
-    Emails(Filter),
-    #[structopt(name="phonenumbers")]
-    PhoneNumbers(Filter),
-}
-
-#[derive(Debug, StructOpt)]
-pub struct Filter {
-    args: Vec<String>,
-}
-
-impl Filter {
-    pub fn parse(&self) -> Result<db::Filter> {
-        db::Filter::parse(&self.args)
-    }
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Target,
 }
 
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
-    let rows = match args {
-        Args::Domains(filter) => scope::<Domain>(rl, &filter),
-        Args::Subdomains(filter) => scope::<Subdomain>(rl, &filter),
-        Args::IpAddrs(filter) => scope::<IpAddr>(rl, &filter),
-        Args::Urls(filter) => scope::<Url>(rl, &filter),
-        Args::Emails(filter) => scope::<Email>(rl, &filter),
-        Args::PhoneNumbers(filter) => scope::<PhoneNumber>(rl, &filter),
+    let rows = match args.subcommand {
+        Target::Domains(filter) => scope::<Domain>(rl, &filter),
+        Target::Subdomains(filter) => scope::<Subdomain>(rl, &filter),
+        Target::IpAddrs(filter) => scope::<IpAddr>(rl, &filter),
+        Target::Urls(filter) => scope::<Url>(rl, &filter),
+        Target::Emails(filter) => scope::<Email>(rl, &filter),
+        Target::PhoneNumbers(filter) => scope::<PhoneNumber>(rl, &filter),
+        Target::Devices(filter) => scope::<Device>(rl, &filter),
+        Target::Networks(filter) => scope::<Network>(rl, &filter),
+        Target::Accounts(filter) => scope::<Account>(rl, &filter),
+        Target::Breaches(filter) => scope::<Breach>(rl, &filter),
+        Target::Images(filter) => scope::<Image>(rl, &filter),
+        Target::Ports(filter) => scope::<Port>(rl, &filter),
     }?;
     term::info(&format!("Updated {} rows", rows));
     Ok(())
 }
 
+#[inline]
 fn scope<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<usize> {
-    rl.db().scope::<T>(&filter.parse()?)
+    T::scope(rl.db(), &filter.parse()?)
 }