Update dependencies

Update scripting documentation

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.11.2"
+version = "0.13.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -9,13 +9,6 @@ categories = ["command-line-utilities"]
 readme = "README.md"
 edition = "2018"
 
-[profile.release]
-# skip lto to avoid compiler bug:
-# https://github.com/kpcyrd/sn0int/issues/77
-# https://github.com/rust-lang/rust/issues/58674
-opt-level = 1
-lto = false
-
 [badges]
 travis-ci = { repository = "kpcyrd/sn0int" }
 
@@ -23,29 +16,39 @@ travis-ci = { repository = "kpcyrd/sn0int" }
 members = ["sn0int-registry/sn0int-common",
            "sn0int-registry"]
 
+[features]
+sqlite-bundled = ["libsqlite3-sys/bundled"]
+
 [dependencies]
-sn0int-common = { version="0.6.0", path="sn0int-registry/sn0int-common" }
-rustyline = "4.0"
+sn0int-common = { version="0.8.0", path="sn0int-registry/sn0int-common" }
+rustyline = "5.0"
 log = "0.4"
-env_logger = "0.6"
+env_logger = "0.7"
 hlua-badtouch = "0.4"
-structopt = "0.2"
+structopt = "0.3"
 failure = "0.1"
-rand = "0.6"
+rand = "0.7"
 colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
 publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
+libsqlite3-sys = "0.12.0"
 chrono = { version = "0.4", features = ["serde"] }
-dirs = "1.0"
-url = "1.7"
+dirs = "2.0"
+url = "2.0"
+percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.10"
+chrootable-https = "0.12"
+rustls = { version="0.16", features=["dangerous_configuration"] }
+webpki = "0.21"
+webpki-roots = "0.18"
+pem = "0.6.0"
 base64 = "0.10"
+data-encoding = "2.1.2"
 kuchiki = "0.7.2"
-serde_urlencoded = "0.5"
+serde_urlencoded = "0.6"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
@@ -54,40 +57,45 @@ ctrlc = "3.1"
 opener = "0.4"
 separator = "0.4"
 maplit = "1.0.1"
-sloppy-rfc4880 = "0.1.2"
+sloppy-rfc4880 = "0.1.5"
 regex = "1.0"
 toml = "0.5"
 maxminddb = "0.13"
 tar = "0.4.17"
 libflate = "0.1.14"
 threadpool = "1.7"
-x509-parser = "0.4.0"
-der-parser = "1.1.0"
-nom = "4.1.1"
+x509-parser = "0.6.0"
+der-parser = "3.0"
 atty = "0.2"
 bufstream = "0.1.4"
 tokio = "0.1.14"
 semver = "0.9"
 bytes = "0.4"
+xml-rs = "0.8"
+bytesize = "1.0"
+ipnetwork = "0.15"
+strum = "0.16"
+strum_macros = "0.16"
 
 digest = "0.8.0"
-hex = "0.3.1"
-bs58 = "0.2.2"
+bs58 = "0.3"
 blake2 = "0.8.0"
 md-5 = "0.8.0"
 sha-1 = "0.8.1"
 sha2 = "0.8.0"
+sha3 = "0.8.0"
+hmac = "0.7"
 
-image = "0.21"
+image = "0.22"
 kamadak-exif = "0.3.1"
 walkdir = "2.2"
-nude = "0.1.0"
+nude = "0.2"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
 #syscallz = { path="../syscallz-rs" }
 syscallz = "0.11"
-nix = "0.13"
+nix = "0.15"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.3.1"
@@ -95,5 +103,5 @@ unveil = "0.2.0"
 
 [dev-dependencies]
 #boxxy = { path = "../boxxy-rs" }
-boxxy = "0.10"
+boxxy = "0.11"
 tempfile = "3.0"

Dockerfile

@@ -1,4 +1,4 @@
-FROM rust
+FROM rust:buster
 RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
@@ -6,7 +6,7 @@ COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian
+FROM debian:buster
 RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int

README.md

@@ -1,4 +1,4 @@
-# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
+# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
 [travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
 [travis]:       https://travis-ci.org/kpcyrd/sn0int
@@ -8,16 +8,20 @@
 [docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
 [irc-img]:      https://img.shields.io/badge/hackint-%23sn0int-blue.svg
 [irc]:          https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int
+[twitter-img]:  https://img.shields.io/badge/twitter-@sn0int-blue.svg
+[twitter]:      https://twitter.com/sn0int
 [mastodon-img]: https://img.shields.io/badge/mastodon-chaos.social-blue.svg
 [mastodon]:     https://chaos.social/@sn0int
 [registry-img]: https://img.shields.io/website/https/sn0int.com.svg?label=registry
 [registry]:     https://sn0int.com/
 
-sn0int is a semi-automatic OSINT framework and package manager. It was built
-for IT security professionals and bug hunters to gather intelligence about a
-given target or about yourself. sn0int is enumerating attack surface by
-semi-automatically processing public information and mapping the results in a
-unified format for followup investigations.
+sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
+package manager. It was built for IT security professionals and bug hunters to
+gather intelligence about a given target or about yourself. sn0int is
+enumerating attack surface by semi-automatically processing public information
+and mapping the results in a unified format for followup investigations.
+
+[ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
 
 Among other things, sn0int is currently able to:
 
@@ -28,6 +32,7 @@ Among other things, sn0int is currently able to:
 - Find somebody's profiles across the internet
 - Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
+- Attempt to bypass cloudflare with shodan
 - Harvest data and images from instagram profiles
 - Scan images for nudity
 
@@ -62,21 +67,36 @@ For everything else please have a look at the [detailed list][1].
 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
-  - [Debian testing/Debian sid/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-testing-debian-sid-kali)
-  - [Ubuntu/Debian stable](https://sn0int.readthedocs.io/en/latest/install.html#ubuntu-debian-stable)
+  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
   - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
   - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
+  - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
+- [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
+  - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
+    - [Archlinux](https://sn0int.readthedocs.io/en/latest/build.html#archlinux)
+    - [Mac OSX](https://sn0int.readthedocs.io/en/latest/build.html#mac-osx)
+    - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/build.html#debian-ubuntu-kali)
+    - [Alpine](https://sn0int.readthedocs.io/en/latest/build.html#alpine)
+    - [OpenBSD](https://sn0int.readthedocs.io/en/latest/build.html#openbsd)
+    - [Gentoo](https://sn0int.readthedocs.io/en/latest/build.html#gentoo)
+    - [Windows](https://sn0int.readthedocs.io/en/latest/build.html#windows)
+  - [Building](https://sn0int.readthedocs.io/en/latest/build.html#building)
 - [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
   - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
   - [Adding something to scope](https://sn0int.readthedocs.io/en/latest/usage.html#adding-something-to-scope)
   - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
   - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
   - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
-- [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
-  - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
+  - [Autonoscope](https://sn0int.readthedocs.io/en/latest/usage.html#autonoscope)
+    - [Domains](https://sn0int.readthedocs.io/en/latest/usage.html#domains)
+    - [IPs](https://sn0int.readthedocs.io/en/latest/usage.html#ips)
+    - [URLs](https://sn0int.readthedocs.io/en/latest/usage.html#urls)
+- [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
+  - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Publish your repo](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-repo)
   - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
@@ -94,10 +114,12 @@ For everything else please have a look at the [detailed list][1].
   - [Accounts](https://sn0int.readthedocs.io/en/latest/structs.html#accounts)
   - [Breaches](https://sn0int.readthedocs.io/en/latest/structs.html#breaches)
   - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
+  - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
+  - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
   - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
-- [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
-- [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
-- [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+    - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
+    - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
+    - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
@@ -113,6 +135,13 @@ For everything else please have a look at the [detailed list][1].
   - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
   - [Diagnosing a sandbox failure](https://sn0int.readthedocs.io/en/latest/sandbox.html#diagnosing-a-sandbox-failure)
 - [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
+  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
+  - [base64_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-decode)
+  - [base64_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-encode)
+  - [base64_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-decode)
+  - [base64_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-encode)
+  - [base32_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-decode)
+  - [base32_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-encode)
   - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
   - [create_blob](https://sn0int.readthedocs.io/en/latest/reference.html#create-blob)
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
@@ -122,13 +151,21 @@ For everything else please have a look at the [detailed list][1].
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
   - [error](https://sn0int.readthedocs.io/en/latest/reference.html#error)
-  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
   - [geoip_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#geoip-lookup)
+  - [hex](https://sn0int.readthedocs.io/en/latest/reference.html#hex)
+  - [hmac_md5](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-md5)
+  - [hmac_sha1](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha1)
+  - [hmac_sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-256)
+  - [hmac_sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-512)
+  - [hmac_sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-256)
+  - [hmac_sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-512)
   - [html_select](https://sn0int.readthedocs.io/en/latest/reference.html#html-select)
   - [html_select_list](https://sn0int.readthedocs.io/en/latest/reference.html#html-select-list)
   - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
   - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
   - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [http_fetch](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch)
+  - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
   - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
@@ -145,11 +182,19 @@ For everything else please have a look at the [detailed list][1].
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
+  - [set_err](https://sn0int.readthedocs.io/en/latest/reference.html#set-err)
   - [sha1](https://sn0int.readthedocs.io/en/latest/reference.html#sha1)
   - [sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-256)
   - [sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-512)
+  - [sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-256)
+  - [sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-512)
   - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [sn0int_time](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time)
+  - [sn0int_time_from](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time-from)
+  - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
+  - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
   - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
   - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
@@ -163,6 +208,10 @@ For everything else please have a look at the [detailed list][1].
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
+  - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
   - [url_escape](https://sn0int.readthedocs.io/en/latest/reference.html#url-escape)
@@ -170,7 +219,11 @@ For everything else please have a look at the [detailed list][1].
   - [url_parse](https://sn0int.readthedocs.io/en/latest/reference.html#url-parse)
   - [url_unescape](https://sn0int.readthedocs.io/en/latest/reference.html#url-unescape)
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
+  - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
+  - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
+  - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
+  - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)
 
 ## Rationale
 

ci/run.sh

@@ -19,8 +19,8 @@ case "$1" in
         cargo test --verbose
         ;;
     windows)
-        export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
-        ci/run.sh "$2"
+        cargo build --verbose --features=sqlite-bundled
+        cargo build --verbose --examples --features=sqlite-bundled
         ;;
     boxxy)
         cargo build --verbose --examples

ci/setup.sh

@@ -5,9 +5,4 @@ case "$1" in
         sudo apt update
         sudo apt install libsqlite3-dev libseccomp-dev
         ;;
-    windows)
-        curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-        7z e sqlite3.zip -y
-        "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
-        ;;
 esac

contrib/html-toc2md.pl

@@ -24,11 +24,11 @@ while (<$r>) {
 # generate new toc
 while (my $line = <$t>) {
     if ($line =~ /toctree-l(\d).*href="([^"]+)">(.+)<\/a/) {
-        my $space = $1;
+        my $space = int($1)-1;
         my $section = $2;
         my $label = $3;
         $label =~ s/([\[\]])/\\$1/g;
-        print $space==2?"  ":"", "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
+        print " " x ($space*2), "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
     }
 }
 print;

docs/build.rst

@@ -0,0 +1,80 @@
+Build from source
+=================
+
+It's generally recommended to `install a package <install.html>`_ if available.
+This section is about building the binary from git.
+
+Install dependencies
+--------------------
+
+You need a recent rust compiler. It's usually recommended to install a rust
+compiler with `rustup <https://rustup.rs/>`_, but if you're system ships the
+most recent compiler in a package that works too. Note that some systems aren't
+fully supported by rustup (like OpenBSD and alpine) and you need to install
+rust from a package in that case.
+
+Archlinux
+~~~~~~~~~
+
+.. code-block:: bash
+
+    $ pacman -S geoip2-database libseccomp publicsuffix-list sqlite
+
+Mac OSX
+~~~~~~~
+
+None.
+
+Debian/Ubuntu/Kali
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+
+.. warning::
+   On a debian based system make sure you've installed rust with rustup.
+
+Alpine
+~~~~~~
+
+.. code-block:: bash
+
+    $ apk add sqlite-dev libseccomp-dev
+
+OpenBSD
+~~~~~~~
+
+.. code-block:: bash
+
+    $ pkg_add sqlite3 geolite2-city geolite2-asn
+
+Gentoo
+~~~~~~
+
+.. code-block:: bash
+
+    emerge --ask sys-libs/libseccomp dev-db/sqlite
+
+Windows
+~~~~~~~
+
+You don't need to install any dependencies on windows, but you need to use a
+different build command in the next section.
+
+Building
+--------
+
+After all dependencies have been installed, simply build the binary:
+
+.. code-block:: bash
+
+    $ cargo build --release
+
+Note that you need a different command on windows:
+
+.. code-block:: bash
+
+    $ cargo build --release --features=sqlite-bundled
+
+After the build finished the binary is located at ``target/release/sn0int``.

docs/index.rst

@@ -36,6 +36,7 @@ Getting Started
    :glob:
 
    install
+   build
    usage
    scripting
    database

docs/install.rst

@@ -1,7 +1,9 @@
 Installation
 ============
 
-If available, please prefer the package shipped by your linux distribution.
+If available, please prefer the package shipped by operating system. If your
+operating system has a package but you're running on older version, please use
+the `build from source <build.html>`_ instructions instead.
 
 Archlinux
 ---------
@@ -17,27 +19,16 @@ Mac OSX
 
     $ brew install sn0int
 
-Debian testing/Debian sid/Kali
-------------------------------
+Debian/Ubuntu/Kali
+------------------
 
 Note that debian `doesn't ship the geoip2-database
 <https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
 download them automatically during the first run.
 
-.. code-block:: bash
-
-    $ apt install build-essential cargo libsqlite3-dev libseccomp-dev publicsuffix
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
-
-Ubuntu/Debian stable
---------------------
-
-cargo in the repos is too old and the build is `going to fail
-<https://github.com/kpcyrd/sn0int/issues/68>`_. You should either install the
-most recent rust version with `rustup <https://rustup.rs/>`_ or use the docker
-instructions instead.
+Using rust+cargo from the repos might work for you, but we only officially
+support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
+at the docker image as an alternative.
 
 .. code-block:: bash
 
@@ -51,39 +42,44 @@ Docker
 
 .. code-block:: bash
 
-    $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
+    $ docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
 
 Alpine
 ------
 
+On alpine edge, with enabled testing repositories:
+
 .. code-block:: bash
 
-    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ apk add sn0int
 
 OpenBSD
 -------
 
+On -current:
+
 .. code-block:: bash
 
-    $ pkg_add git cargo sqlite3 geolite2-city geolite2-asn
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ pkg_add sn0int
+
+Gentoo
+------
+
+.. code-block:: bash
+
+    layman -f -o https://raw.githubusercontent.com/kpcyrd/overlay/master/overlay.xml -a kpcyrd-overlay
+    emerge --ask net-analyzer/sn0int
 
 Windows
 -------
 
-This is not recommended and only passively maintained. Please prefer linux in a virtual machine if needed.
+This is not recommended and only passively maintained. Please prefer linux in a
+virtual machine if needed.
+
+Make sure rust is installed and setup.
 
 .. code-block:: bash
 
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-    $ 7z e sqlite3.zip -y
-    $ "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
-    $ export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
-    $ cargo install -f --path .
+    $ cargo install -f --path . --features=sqlite-bundled

docs/reference.rst

@@ -1,6 +1,87 @@
 Function reference
 ==================
 
+asn_lookup
+----------
+
+Run an ASN lookup for a given ip address. The function returns ``asn`` and
+``as_org``. This function may fail.
+
+.. code-block:: lua
+
+    lookup = asn_lookup('1.1.1.1')
+    if last_err() then return end
+
+base64_decode
+-------------
+
+Decode a base64 string with the default alphabet+padding.
+
+.. code-block:: lua
+
+    base64_decode("ww==")
+
+base64_encode
+-------------
+
+Encode a binary array with base64 and the default alphabet+padding.
+
+.. code-block:: lua
+
+    base64_encode("\x00\xff")
+
+base64_custom_decode
+--------------------
+
+Decode a base64 string with custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- base64
+    base64_custom_decode('b2hhaQ==', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '=')
+    -- base64 no padding
+    base64_custom_decode('b2hhaQ', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '')
+    -- base64 url safe
+    base64_custom_decode('b2hhaQ==', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', '=')
+
+base64_custom_encode
+--------------------
+
+Encode a binary array with base64 and custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- base64
+    base64_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '=')
+    -- base64 no padding
+    base64_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '')
+    -- base64 url safe
+    base64_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', '=')
+
+base32_custom_decode
+--------------------
+
+Decode a base32 string with custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- rfc-4648 base32
+    base32_custom_decode('N5UGC2I=', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567', '=')
+    -- z-base-32
+    base32_custom_decode('p7wgn4e', 'ybndrfg8ejkmcpqxot1uwisza345h769', '')
+
+base32_custom_encode
+--------------------
+
+Encode a binary array with base32 and custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- rfc-4648 base32
+    x = base32_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567', '=')
+    -- z-base-32
+    x = base32_custom_encode('ohai', 'ybndrfg8ejkmcpqxot1uwisza345h769', '')
+
 clear_err
 ---------
 
@@ -35,6 +116,14 @@ for ``DATETIME`` database fields.
 
     now = datetime()
 
+.. note::
+    This format is sn0int specific, to get the current time for scripting use
+    time_unix_ instead.
+
+.. warning::
+    This function is going to be deprecated at some point. Prefer sn0int_time_
+    for new scripts.
+
 db_add
 ------
 
@@ -59,7 +148,7 @@ ttl.
 .. code-block:: lua
 
     -- this link is valid for 2min
-    domain_id = db_add('network-device', {
+    domain_id = db_add_ttl('network-device', {
         network_id=1,
         device_id=13,
     }, 120)
@@ -135,17 +224,6 @@ Log an error to the terminal.
 
     error('ohai')
 
-asn_lookup
-----------
-
-Run an ASN lookup for a given ip address. The function returns ``asn`` and
-``as_org``. This function may fail.
-
-.. code-block:: lua
-
-    lookup = asn_lookup('1.1.1.1')
-    if last_err() then return end
-
 geoip_lookup
 ------------
 
@@ -166,6 +244,69 @@ This function may fail.
     lookup = geoip_lookup('1.1.1.1')
     if last_err() then return end
 
+hex
+---
+
+Hex encode a list of bytes.
+
+.. code-block:: lua
+
+    hex("\x6F\x68\x61\x69\x0A\x00")
+
+hmac_md5
+--------
+
+Calculate an hmac with md5. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_md5("secret", "my authenticated message")
+
+hmac_sha1
+---------
+
+Calculate an hmac with sha1. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha1("secret", "my authenticated message")
+
+hmac_sha2_256
+-------------
+
+Calculate an hmac with sha2_256. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha2_256("secret", "my authenticated message")
+
+hmac_sha2_512
+-------------
+
+Calculate an hmac with sha2_512. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha2_512("secret", "my authenticated message")
+
+hmac_sha3_256
+-------------
+
+Calculate an hmac with sha3_256. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha3_256("secret", "my authenticated message")
+
+hmac_sha3_512
+-------------
+
+Calculate an hmac with sha3_512. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha3_512("secret", "my authenticated message")
+
 html_select
 -----------
 
@@ -225,6 +366,9 @@ options are set. The following options are available:
 ``into_blob``
   If true, the response body is stored in blob storage and a blob reference is
   returned as ``blob`` instead of the full body.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
 
 This function may fail.
 
@@ -238,7 +382,7 @@ This function may fail.
     })
     resp = http_send(req)
     if last_err() then return end
-    if resp["status"] ~= 200 then return "invalid status code" end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
 
 http_send
 ---------
@@ -266,7 +410,46 @@ the following keys:
     })
     resp = http_send(req)
     if last_err() then return end
-    if resp["status"] ~= 200 then return "invalid status code" end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+
+http_fetch
+----------
+
+This does an http_send_ and also automatically validate the status code.
+
+.. note::
+   You almost always want this when setting the ``into_blob`` option since this
+   function validates the status code *before* inserting the response body into
+   blob storage.
+
+.. code-block:: lua
+
+    -- short form
+    data = http_fetch(req)
+    if last_err() then return end
+
+    -- long form
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+
+http_fetch_json
+---------------
+
+Identical to http_fetch_ but also automatically parses the response body as json.
+
+.. code-block:: lua
+
+    -- short form
+    data = http_fetch_json(req)
+    if last_err() then return end
+
+    -- long form
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+    data = json_decode(resp['text'])
+    if last_err() then return end
 
 img_load
 --------
@@ -388,7 +571,8 @@ Same as pgp_pubkey_armored_, but without the unarmor step.
 pgp_pubkey_armored
 ------------------
 
-Extract uids out of a rfc 4880 pgp public key. This function may fail.
+Extract ``uids``, ``sigs`` and the ``fingerprint`` out of an rfc 4880 pgp
+public key. This function may fail.
 
 .. code-block:: lua
 
@@ -492,6 +676,41 @@ Same as regex_find_, but returns all matches.
     print(m[3][1] == 'ef')
     print(m[3][2] == 'f')
 
+semver_match
+------------
+
+Compare a version to a version requirement. This can be used with
+sn0int_version_ to test for certain features or behavior.
+
+.. code-block:: lua
+
+    semver_match('=0.11.2', sn0int_version())
+    semver_match('>0.11.2', sn0int_version())
+    semver_match('<0.11.2', sn0int_version())
+    semver_match('~0.11.2', sn0int_version())
+    semver_match('^0.11.2', sn0int_version())
+    semver_match('0.11.2', sn0int_version()) -- synonym for ^0.11.2
+    semver_match('<=0.11.2', sn0int_version())
+    semver_match('>=0.11.2', sn0int_version())
+    semver_match('>=0.4.0, <=0.10.0', sn0int_version())
+
+set_err
+-------
+
+Manipulate the global error object. If you want to exit the main ``run``
+function with an error you can simply return a string, but those are difficult
+to propagate through functions. ``set_err`` specifically assigns an error to
+the global error object that are also used by all other rust functions.
+
+.. code-block:: lua
+
+    function foo()
+        set_err("something failed")
+    end
+
+    foo()
+    if last_err() then return end
+
 sha1
 ----
 
@@ -519,6 +738,24 @@ Hash a byte array with sha2_512 and return the results as bytes.
 
     hex(sha2_512("\x00\xff"))
 
+sha3_256
+--------
+
+Hash a byte array with sha3_256 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha3_256("\x00\xff"))
+
+sha3_512
+--------
+
+Hash a byte array with sha3_512 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha3_512("\x00\xff"))
+
 sleep
 -----
 
@@ -529,14 +766,88 @@ only used for debugging.
 
     sleep(1)
 
+sn0int_time
+-----------
+
+Return current time in UTC. This function is suitable to determine datetimes
+for ``DATETIME`` database fields.
+
+.. code-block:: lua
+
+    now = sn0int_time()
+
+.. note::
+    This format is sn0int specific, to get the current time for scripting use
+    time_unix_ instead.
+
+sn0int_time_from
+----------------
+
+Identical to sn0int_time_ but uses a unix timestamp in seconds instead of the
+current time. This function is compatible with time_unix_ and strptime_.
+
+.. code-block:: lua
+
+    time = sn0int_time_from(1567931337)
+
+sn0int_version
+--------------
+
+Get the current sn0int version string. This can be used with semver_match_ to
+test for certain features or behavior.
+
+.. code-block:: lua
+
+    info(sn0int_version())
+
 sock_connect
 ------------
 
 Create a tcp connection.
 
+The following options are available:
+
+``tls``
+  Set to true to enable tls (certificates are validated)
+``sni_value``
+  Instead of the host argument, use a custom string for the sni extension.
+``disable_tls_verify``
+  **Danger**: disable tls verification. This disables all security on the
+  connection. Note that sn0int is still rather strict, you're going to run into
+  issues if you need support for insecure ciphers.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+
 .. code-block:: lua
 
-    sock = sock_connect("127.0.0.1", 1337)
+    sock = sock_connect("127.0.0.1", 1337, {
+        tls=true,
+    })
+
+sock_upgrade_tls
+----------------
+
+Take an existing tcp connection and start a tls handshake. The options are the
+same as sock_connect_ but the ``tls`` value is always assumed to be true.
+
+The sni value needs to be set specifically, otherwise the sni extension is
+disabled.
+
+Using this function specifically returns some extra information that is
+discarded when using sock_connect_ directly with ``tls=true``.
+
+.. code-block:: lua
+
+    sock = sock_connect("127.0.0.1", 1337, {})
+    if last_err() then return end
+
+    tls = sock_upgrade_tls(sock, {
+        sni_value='example.com',
+    })
+    if last_err() then return end
+
+    info(tls)
 
 sock_send
 ---------
@@ -660,6 +971,50 @@ Read a line from stdin. The final newline is not removed.
 .. note::
    This only works with `sn0int run --stdin`.
 
+.. TODO: add stdin_read_line and deprecate stdin_readline
+
+stdin_read_to_end
+-----------------
+
+Read stdin until EOF as a utf-8 string.
+
+.. code-block:: lua
+
+    stdin_read_to_end()
+
+.. note::
+   This only works with `sn0int run --stdin`.
+
+strftime
+--------
+
+Format a timestamp generated with time_unix_ into a date, see `strftime rules`_.
+
+.. code-block:: lua
+
+    t = strftime('%d/%m/%Y %H:%M', 1558584994)
+
+strptime
+--------
+
+Parse a date into a unix timestamp, see `strftime rules`_.
+
+.. code-block:: lua
+
+    t = strptime('%d/%m/%Y %H:%M', '23/05/2019 04:16')
+
+.. _strftime rules: https://docs.rs/chrono/0.4.6/chrono/format/strftime/index.html
+
+time_unix
+---------
+
+Get the current time as seconds since ``January 1, 1970 0:00:00 UTC``, also
+known as UNIX timestamp. This timestamp can be formated using strftime_.
+
+.. code-block:: lua
+
+    now = time_unix()
+
 url_decode
 ----------
 
@@ -749,6 +1104,27 @@ Decodes a list of bytes/numbers into a string. This function might fail.
     if last_err() then return end
     print(x == 'AAAA')
 
+warn
+----
+
+Log a warning to the terminal.
+
+.. code-block:: lua
+
+    warn('ohai')
+
+warn_once
+---------
+
+Log a warning to the terminal once. This can be used to print a warning to the
+user without printing the same warning for each struct we're processing during
+a ``run`` execution.
+
+.. code-block:: lua
+
+    warn_once('ohai')
+    warn_once('ohai')
+
 x509_parse_pem
 --------------
 
@@ -783,3 +1159,35 @@ Parse a pem encoded certificate. This function might fail.
     ]])
     if last_err() then return end
     print(x)
+
+xml_decode
+----------
+
+Decode a lua value from an xml document.
+
+.. code-block:: lua
+
+    x = xml_decode('<body><foo fizz="buzz">bar</foo></body>')
+    if last_err() then return end
+
+    body = x['children'][1]
+    foo = body['children'][1]
+
+    print(foo['attrs']['fizz'])
+    print(foo['text'])
+
+xml_named
+---------
+
+Get a named child element from a parent element.
+
+.. code-block:: lua
+
+    x = xml_decode('<body><foo fizz="buzz">bar</foo></body>')
+    if last_err() then return end
+
+    body = x['children'][1]
+    foo = xml_named(body, 'foo')
+    if foo ~= nil then
+        print(foo)
+    end

docs/scripting.rst

@@ -1,11 +1,11 @@
-Scripting
-=========
+Writing your first module
+=========================
 
 Scripting is the core feature in sn0int. It's not strictly required, but if you
 want to write your own modules, this section is for you.
 
-Write your first module
------------------------
+Creating a repository
+---------------------
 
 It's highly recommended to use a VCS for development, so let's start by setting
 that up. We're going to assume you store your repos in ``~/repos`` but you're
@@ -14,6 +14,12 @@ free to change that to something else::
     $ git init ~/repos/sn0int-modules
     $ cd ~/repos/sn0int-modules
 
+.. note::
+   If you're using github you can also create a repo from the `module repo
+   template`_.
+
+.. _module repo template: https://github.com/sn0int/sn0int-modules
+
 We need to add this folder to the sn0int config file so it's correctly detected
 when starting sn0int. Open the `config file <config.html>`_ in your prefered
 editor. Note that the file does not exist by default and the path is different
@@ -22,7 +28,7 @@ with::
 
     $ vim ~/.config/sn0int.toml
 
-Add the follwing::
+Add the following::
 
     [namespaces]
     your_github_name = "~/repos/sn0int-modules"
@@ -90,21 +96,26 @@ database.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
-        print(subdomain)
+        info(subdomain)
     end
 
-Combined with the header we wrote previously we can already execute this
-module. Make sure you've added a domain to scope with ``add domain
-example.com``, save your file and run it like this::
+This is already enough to execute it. Make sure you've added a domain to scope
+with ``add domain example.com``, save your file and run it like this::
 
     sn0int run -f ./first.lua
 
-We should see some output by our print function.
+We should see some output by our info function.
 
 .. note::
-   ``print`` is useful for development but must be removed before publishing.
+   ``info`` is useful for development but you usually want your module to run
+   quietly, so before publishing either remove it or replace it with ``debug``.
 
 Next, we want to actually resolve that name, we're going to use the ``dns``
 function for that. This function takes a name and a query type and returns a
@@ -114,6 +125,11 @@ truth-y.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
@@ -122,16 +138,22 @@ truth-y.
         })
         if last_err() then return end
 
-        print(records)
+        info(records)
     end
 
 If you run your module again you're going to see some output, either
 ``{"answers":[somedata],"error":null}`` or
-``{"answers":[],"error":"NXDomain"}``. We decide that we add the subdomain to
-our scope and set it to resolvable if ``error`` is ``nil``.
+``{"answers":[],"error":"NXDomain"}``. If the dns reply doesn't indicate an
+error this means the subdomain exists and we can add it to our database with
+``resolvable`` being set to ``true``.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
@@ -152,41 +174,20 @@ our scope and set it to resolvable if ``error`` is ``nil``.
 .. hint::
    See the database section to understand how the database works in detail.
 
-If we execute our module one more time it's going to log that it discovered a
-subdomain, if it doesn't, try adding more domains to scope. Note that this only
-happens the first time. Modules that don't discover anything or don't discover
-anything new exit silently.
-
-After putting everything together, our final module looks like this:
-
-.. code-block:: lua
-
-    -- Description: ohai wurld
-    -- Version: 0.1.0
-    -- Source: domains
-    -- License: GPL-3.0
-
-    function run(arg)
-        subdomain = 'www.' .. arg['value']
-
-        records = dns(subdomain, {
-            record='A'
-        })
-        if last_err() then return end
-
-        if records['success'] ~= nil then
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=subdomain,
-                resolvable=true,
-            })
-        end
-    end
+If we execute our finished module one more time it's going to log that it
+discovered a subdomain, if it doesn't, try adding more domains to scope. Note
+that this only happens the first time. Modules that don't discover anything or
+don't discover anything new exit silently.
 
 There's still some room for improvement, for example, since we already resolved
 that record, we could also add the ip address to the scope and link it to the
 subdomain we added.
 
+.. hint::
+   For debugging purposes you can increase the verbosity with ``sn0int run -v``
+   so database operations are logged even if nothing was changed, or with
+   ``sn0int run -vv`` to enable ``debug()`` output.
+
 Publish your module
 -------------------
 
@@ -205,6 +206,24 @@ Afterwards publish your module with::
 
     sn0int publish ./first.lua
 
+Please also make sure you publish your repository to github so other people can
+submit pull requests. The recommended repository location is::
+
+    https://github.com/<your-username>/sn0int-modules
+
+Publish your repo
+-----------------
+
+It is highly recommended to publish your repository on github so people can
+file issues and pull requests for your module. If you've been following along
+with the github template you can simply commit your changes and push them.
+
+Your repository would look like one of these:
+
+- https://github.com/kpcyrd/sn0int-modules
+- https://github.com/ysf/sn0int-modules
+- https://github.com/cybiere/sn0int-modules
+
 Reading data from stdin
 -----------------------
 

docs/structs.rst

@@ -145,6 +145,8 @@ connected to.
     Latitude of the networks location.
 ``longitude``
     Longitude of the networks location.
+``description``
+    A human readable description in case the value is a technical identifier.
 
 Accounts
 --------
@@ -166,6 +168,12 @@ A users account or profile on a webservice, like github or instagram.
     The url of the public profile if available.
 ``last_seen``
     The last time this account has been active/online.
+``birthday``
+    The users birthday set on the account.
+``phonenumber``
+    The phonenumber associated with the account.
+``profile_pic``
+    The blob identifier of the users current profile picture.
 
 Breaches
 --------
@@ -207,6 +215,50 @@ Images
 ``phash``
     The DCT (pHash) perceptual hash.
 
+Ports
+-----
+
+The status of a port on an ip address.
+
+``ip_addr_id``
+    The numeric id of an ipaddr struct.
+``ip_addr``
+    The actual ipaddr.
+``port``
+    The port number.
+``status``
+    The status of the port, either ``open`` or ``closed``.
+``banner``
+    The service banner we discovered on this port.
+``service``
+    The service that is running on this port.
+``version``
+    The version of the service running on this port.
+
+Netblocks
+---------
+
+A netblock is a network address range that has been allocated to an individual,
+organization or company. Those are commonly found when running whois lookups on
+an ip address.
+
+Consider the following example: Running a whois lookup on ``140.82.118.4`` (one
+of the addresses currently in use by github) returns that this address belongs
+to the netrange ``140.82.112.0 - 140.82.127.255``, so the netblock in this case
+is ``140.82.112.0/20``.
+
+``family``
+    This is either ``4`` or ``6`` and populated automatically.
+``value``
+    This is the network range in CIDR notation.
+``asn``
+    The number of the autonomous system this network belongs to.
+``as_org``
+    The organization of the autonomous system this network belongs to.
+``description``
+    This field isn't strictly defined and meant to be used as a human
+    meaningful name if available.
+
 Relations
 ---------
 

docs/usage.rst

@@ -256,3 +256,85 @@ You can reverse this using the scope command::
 .. hint::
    All entities have this field, you can refer to it in queries using
    ``unscoped=1``.
+
+Autonoscope
+-----------
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+~~~~~~~
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+~~~
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+~~~~
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

migrations/2019-05-30-142142_ports/down.sql

@@ -0,0 +1 @@
+DROP TABLE ports;

migrations/2019-05-30-142142_ports/up.sql

@@ -0,0 +1,17 @@
+CREATE TABLE ports (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);

migrations/2019-07-04-233515_autonoscope/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE autonoscope;

migrations/2019-07-04-233515_autonoscope/up.sql

@@ -0,0 +1,8 @@
+-- Your SQL goes here
+CREATE TABLE autonoscope (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    object VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    scoped BOOLEAN NOT NULL,
+    CONSTRAINT autonoscope_unique UNIQUE (object, value)
+);

migrations/2019-07-27-152215_netblocks/down.sql

@@ -0,0 +1 @@
+DROP TABLE netblocks;

migrations/2019-07-27-152215_netblocks/up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE netblocks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    asn INTEGER,
+    as_org VARCHAR,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2019-08-11-073709_misc-fields/down.sql

@@ -0,0 +1,41 @@
+PRAGMA foreign_keys=off;
+
+-- accounts
+CREATE TABLE _accounts_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);
+
+INSERT INTO _accounts_new (id, value, service, username, displayname, email, url, last_seen, unscoped)
+    SELECT id, value, service, username, displayname, email, url, last_seen, unscoped
+    FROM accounts;
+
+DROP TABLE accounts;
+ALTER TABLE _accounts_new RENAME TO accounts;
+
+-- networks
+CREATE TABLE _networks_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+INSERT INTO _networks_new (id, value, unscoped, latitude, longitude)
+    SELECT id, value, unscoped, latitude, longitude
+    FROM networks;
+
+DROP TABLE networks;
+ALTER TABLE _networks_new RENAME TO networks;
+
+PRAGMA foreign_keys=on;

migrations/2019-08-11-073709_misc-fields/up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE accounts ADD COLUMN phonenumber VARCHAR;
+ALTER TABLE accounts ADD COLUMN profile_pic VARCHAR;
+ALTER TABLE accounts ADD COLUMN birthday VARCHAR;
+ALTER TABLE networks ADD COLUMN description VARCHAR;

modules/dev/arp-scan.lua

@@ -1,45 +0,0 @@
--- Description: Parse arp-scan output
--- Version: 0.3.0
--- License: GPL-3.0
-
--- sudo arp-scan -qglI wlp3s0
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        m = regex_find('(.+)\t(.+)', x)
-        if m ~= nil then
-            ipaddr = m[2]
-            mac = m[3]
-            now = datetime()
-
-            device_id = db_add('device', {
-                value=mac,
-                last_seen=now,
-            })
-            if last_err() then return end
-
-            db_add_ttl('network-device', {
-                network_id=network_id,
-                device_id=device_id,
-                ipaddr=ipaddr,
-                last_seen=now,
-            }, 300)
-            if last_err() then return end
-        end
-    end
-end

modules/dev/asn.lua

@@ -1,16 +0,0 @@
--- Description: Run a asn lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = asn_lookup(arg['value'])
-    if last_err() then return end
-
-    if arg['asn'] ~= lookup['asn'] or arg['as_org'] ~= lookup['as_org'] then
-        db_update('ipaddr', arg, {
-            asn=lookup['asn'],
-            as_org=lookup['as_org'],
-        })
-    end
-end

modules/dev/axfr.lua

@@ -1,145 +0,0 @@
--- Description: Try a zone transfer for subdomains
--- Version: 0.3.0
--- Source: domains
--- License: GPL-3.0
-
-function strip_root_dot(name)
-    local m = regex_find("(.+)\\.$", name)
-    if last_err() then return end
-
-    if m == nil then
-        return name
-    else
-        return m[2]
-    end
-end
-
-function add_pointer(name)
-    -- select psl+1
-    local domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    local subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-end
-
-function iter_axfr(zone, arg)
-    debug(arg)
-
-    local name = arg[1]
-    local r = arg[2]
-
-    -- select psl+1
-    local domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    local subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-
-    -- this is a A record
-    if r['A'] ~= nil then
-        -- add the name and ip
-        local ipaddr_id = db_add('ipaddr', {
-            family='4',
-            value=r['A'],
-        })
-        if last_err() then return end
-
-        db_add('subdomain-ipaddr', {
-            subdomain_id=subdomain_id,
-            ip_addr_id=ipaddr_id,
-        })
-        if last_err() then return end
-    end
-
-    if r['CNAME'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['CNAME'])
-        add_pointer(name)
-    end
-
-    if r['NS'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['NS'])
-        add_pointer(name)
-    end
-
-    if r['MX'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['MX'][2])
-        add_pointer(name:lower())
-    end
-end
-
-function iter_a(zone, arg)
-    if arg == nil then return end
-
-    debug('nameserver: ' .. arg)
-    local records = dns(zone, {
-        record='AXFR',
-        nameserver=arg .. ':53',
-        tcp=true,
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        iter_axfr(zone, records[i])
-        if last_err() then return end
-    end
-end
-
-function iter_ns(zone, arg)
-    if arg == nil then return end
-
-    local records = dns(arg, {
-        record='A',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        r = records[i][2]
-        iter_a(zone, r['A'])
-        if last_err() then return end
-    end
-end
-
-function run(arg)
-    local records = dns(arg['value'], {
-        record='NS',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        local r = records[i][2]
-        iter_ns(arg['value'], r['NS'])
-        if last_err() then return end
-    end
-end

modules/dev/cname-harvest.lua

@@ -1,48 +0,0 @@
--- Description: Query for CNAMES to find subdomains
--- Version: 0.3.0
--- Source: subdomains
--- License: GPL-3.0
-
-function iter(r)
-    if r == nil then
-        return
-    end
-
-    m = regex_find("(.+)\\.$", r)
-    if last_err() then return end
-
-    if m == nil then
-        return
-    end
-    r = m[2]
-
-    domain = psl_domain_from_dns_name(r)
-    if last_err() then return end
-
-    domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=r,
-        })
-        if last_err() then return end
-    end
-end
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        r = records[i][2]
-        iter(r['CNAME'])
-        if last_err() then return end
-    end
-end

modules/dev/ctlogs.lua

@@ -1,101 +0,0 @@
--- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.5.0
--- Source: domains
--- License: GPL-3.0
-
-function each_name(name)
-    local domain_id, psl_domain
-
-    if seen[name] == 1 then
-        return
-    end
-    seen[name] = 1
-    debug(name)
-
-    if name:find('*.') == 1 then
-        -- ignore wildcard domains
-        return
-    end
-
-    -- the cert might be valid for subdomains that do not belong to the
-    -- domain we started with
-    psl_domain = psl_domain_from_dns_name(name)
-    domain_id = domains[psl_domain]
-    if domain_id == nil then
-        if any_domain then
-            -- unknown domains should be added to database
-            domain_id = db_add('domain', {
-                value=psl_domain,
-            })
-        else
-            -- only use domains that are already in scope
-            domain_id = db_select('domain', psl_domain)
-        end
-
-        -- if we didn't get a valid id, skip
-        if domain_id == nil then
-            return
-        end
-
-        domains[psl_domain] = domain_id
-    end
-
-    db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-end
-
-function run(arg)
-    full = getopt('full') ~= nil
-    any_domain = getopt('any-domain') ~= nil
-
-    domains = {}
-    domains[arg['value']] = arg['id']
-
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://crt.sh/', {
-        query={
-            q='%.' .. arg['value'],
-            output='json'
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    certs = json_decode(resp['text'])
-    if last_err() then return end
-
-    seen = {}
-
-    for i=1, #certs do
-        c = certs[i]
-        debug(c)
-
-        if full then
-            -- fetch certificate
-            id = c['min_cert_id']
-            req = http_request(session, 'GET', 'https://crt.sh/', {
-                query={
-                    d=id .. '', -- TODO: find nicer way for tostring
-                }
-            })
-            resp = http_send(req)
-            if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-            -- iterate over all valid names
-            crt = x509_parse_pem(resp['text'])
-            if last_err() then return end
-            names = crt['valid_names']
-
-            for j=1, #names do
-                each_name(names[j])
-            end
-        else
-            each_name(c['name_value'])
-        end
-    end
-end

modules/dev/ddwrt-dhcp.lua

@@ -1,88 +0,0 @@
--- Description: Export dhcp leases from ddwrt webinterface
--- Version: 0.2.0
--- License: GPL-3.0
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    skip_redacted = not getopt('use-redacted')
-
-    router = getopt('router') -- http://192.0.2.1/
-    if not router then
-        return 'router option is missing (http://192.0.2.1/)'
-    end
-    username = getopt('user')
-    password = getopt('password')
-
-    options = {}
-    if username and password then
-        options['basic_auth'] = {username, password}
-    end
-
-    -- request status page
-    session = http_mksession()
-    url = url_join(router, '/Info.live.htm')
-    req = http_request(session, 'GET', url, options)
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then
-        return 'http error: ' .. resp['status']
-    end
-
-    txt = resp['text']
-    debug(txt)
-
-    -- extract leases from response
-    dhcp_section = regex_find('\\{dhcp_leases:: ([^\\}]+)\\}', txt)
-    if last_err() then return end
-    if not dhcp_section then
-        return 'Failed to get dhcp lease section'
-    end
-
-    leases = regex_find_all('\'([^\']+)\',\'([^\']+)\',\'([^\']+)\',\'[^\']+\',\'[^\']+\'', dhcp_section[2])
-    if last_err() then return end
-
-    now = datetime()
-
-    -- add devices to database
-    for i=1, #leases do
-        local hostname = leases[i][2]
-        local ipaddr = leases[i][3]
-        local macaddr = leases[i][4]
-
-        debug({
-            hostname=hostname,
-            ipaddr=ipaddr,
-            macaddr=macaddr,
-        })
-
-        if skip_redacted and macaddr:match('^xx:xx:') then
-            info('Skipping redacted macaddr')
-        else
-            local device = {
-                value=macaddr,
-                last_seen=now,
-            }
-            if hostname ~= '*' then
-                device['hostname'] = hostname
-            end
-
-            local device_id = db_add('device', device)
-
-            db_add_ttl('network-device', {
-                network_id=network_id,
-                device_id=device_id,
-                ipaddr=ipaddr,
-                last_seen=now,
-            }, 120)
-        end
-    end
-end

modules/dev/dns-ns.lua

@@ -1,51 +0,0 @@
--- Description: Add a domains NS records to scope
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: domains
-
-function strip_root_dot(name)
-    local m = regex_find("(.+)\\.$", name)
-    if last_err() then return end
-
-    if m == nil then
-        return name
-    else
-        return m[2]
-    end
-end
-
-function each(r)
-    local name = strip_root_dot(r)
-    local domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    local subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-end
-
-function run(arg)
-    local records = dns(arg['value'], {
-        record='NS',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        local r = records[i][2]
-        debug(r)
-        each(r['NS'])
-        if last_err() then return end
-    end
-end

modules/dev/dns-ptr.lua

@@ -1,30 +0,0 @@
--- Description: Run reverse dns lookups
--- Version: 0.2.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    if arg['family'] == '4' then
-        m = regex_find('^(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)$', arg['value'])
-
-        q = m[5] .. '.' .. m[4] .. '.' .. m[3] .. '.' .. m[2] .. '.in-addr.arpa'
-        debug('Resolving: ' .. q)
-
-        records = dns(q, {
-            record='PTR',
-        })
-        if last_err() then return end
-        if records['error'] ~= nil then return end
-        records = records['answers']
-
-        for i=1, #records do
-            r = records[i][2]
-            if r['PTR'] then
-                db_update('ipaddr', arg, {
-                    reverse_dns=r['PTR'],
-                })
-                if last_err() then return end
-            end
-        end
-    end
-end

modules/dev/dns-resolve.lua

@@ -1,41 +0,0 @@
--- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.3.0
--- Source: subdomains
--- License: GPL-3.0
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    -- update subdomain
-    resolvable = records['error'] == nil
-    if arg['resolvable'] ~= resolvable then
-        -- TODO: pass arg to function as well
-        db_update('subdomain', arg, {
-            resolvable=resolvable
-        })
-    end
-
-    if not resolvable then
-        return
-    end
-
-    records = records['answers']
-
-    for i=1, #records do
-        r = records[i][2]
-        if r['A'] ~= nil then
-            ipaddr_id = db_add('ipaddr', {
-                family='4',
-                value=r['A'],
-            })
-            if last_err() then return end
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=arg['id'],
-                ip_addr_id=ipaddr_id,
-            })
-            if last_err() then return end
-        end
-    end
-end

modules/dev/exif.lua

@@ -1,12 +0,0 @@
--- Description: Extract exif data from images
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: images
-
-function run(arg)
-    exif = img_exif(arg['value'])
-    if last_err() then return end
-    debug(exif)
-
-    db_update('image', arg, exif)
-end

modules/dev/geoip.lua

@@ -1,10 +0,0 @@
--- Description: Run a geoip lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = geoip_lookup(arg['value'])
-    if last_err() then return end
-    db_update('ipaddr', arg, lookup)
-end

modules/dev/git-webroot.lua

@@ -1,28 +0,0 @@
--- Description: Search for git checkouts in webroot
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    url = url_join(arg['value'], '.git/HEAD')
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    reply = http_send(req)
-    if last_err() then return end
-
-    if reply['status'] ~= 200 then
-        return
-    end
-
-    if not regex_find('^ref: ', reply['text']) then
-        return
-    end
-
-    db_add('url', {
-        subdomain_id=arg['subdomain_id'],
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-    })
-end

modules/dev/github.lua

@@ -1,107 +0,0 @@
--- Description: Collect data from github profiles
--- Version: 0.2.0
--- Source: accounts:github.com
--- License: GPL-3.0
-
-function api_get(url)
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    -- TODO: set_error(?)
-    if resp['status'] == 403 then return 'ratelimit exceeded' end
-    if resp['status'] ~= 200 then return 'invalid status code' end
-
-    local data = json_decode(resp['text'])
-    if last_err() then return end
-    return data
-end
-
-function import_gpg(url)
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-
-    local key = pgp_pubkey_armored(resp['text'])
-    if not key['uids'] then return end
-
-    for i=1, #key['uids'] do
-        local k = key['uids'][i]
-        debug(k)
-        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", k)
-        if m then
-            db_add('email', {
-                value=m[3],
-                displayname=m[2],
-            })
-        end
-    end
-end
-
-function scan4email(username)
-    local url = 'https://api.github.com/users/' .. username .. '/repos'
-    local repos = api_get(url)
-    if last_err() then return end
-
-    -- XXX: 'https://api.github.com/users/' .. username .. '/events/public?page=0&per_page=100' is faster but less accurate
-
-    for i=1, #repos do
-        local repo = repos[i]
-        debug(repo)
-        local commits = api_get(repo['url'] .. '/commits')
-        if last_err() then return end
-
-        for j=1, #commits do
-            local commit = commits[j]
-            debug(commit)
-
-            if commit['author'] and commit['author']['login'] == username then
-                local name = commit['commit']['author']['name']
-                local email = commit['commit']['author']['email']
-                db_add('email', {
-                    value=email,
-                    displayname=name,
-                })
-                return email
-            end
-
-            if commit['committer'] and commit['committer']['login'] == username then
-                local name = commit['commit']['committer']['name']
-                local email = commit['commit']['committer']['email']
-                db_add('email', {
-                    value=email,
-                    displayname=name,
-                })
-                return email
-            end
-        end
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    local url = 'https://api.github.com/users/' .. arg['username']
-
-    local data = api_get(url)
-    if last_err() then return end
-    debug(data)
-
-    -- company = data['company']
-    -- location = data['location']
-    -- homepage = data['blog']
-
-    url = 'https://github.com/' .. arg['username'] .. '.gpg'
-    import_gpg(url)
-    if last_err() then return end
-
-    local email = data['email']
-    if not email and not arg['email'] then
-        email = scan4email(arg['username'])
-        if last_err() then return end
-    end
-
-    db_update('account', arg, {
-        url=data['html_url'],
-        displayname=data['name'],
-        email=email,
-    })
-end

modules/dev/hackertarget-subdomains.lua

@@ -1,27 +0,0 @@
--- Description: Query hackertarget for subdomains of a domain
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.hackertarget.com/hostsearch/', {
-        query={
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    m = regex_find_all("([^,]+),.+\\n?", resp['text'])
-
-    for i=1, #m do
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=m[i][2]
-        })
-    end
-end

modules/dev/home-assistant-devices.lua

@@ -1,56 +0,0 @@
--- Description: Query device location from home assistant
--- Version: 0.1.0
--- License: GPL-3.0
--- Keyring-Access: home-assistant
-
-function run()
-    -- parsing options
-    instance = getopt('instance')
-    if not instance then
-        return 'instance option is missing'
-    end
-
-    host = url_parse(instance)
-    if last_err() then return end
-    host = host['host']
-
-    entity = getopt('entity')
-    if not entity then
-        return 'entity option is missing'
-    end
-
-    -- fetching credentials
-    creds = keyring('home-assistant:' .. host)
-    if creds[1] == nil then
-        profile = url_join(instance, 'profile')
-        return 'missing home-assistant:' .. host .. ' Long-Lived Access Token, open ' .. profile
-    end
-    token = creds[1]['secret_key']
-
-    headers = {}
-    headers['Authorization'] = 'Bearer ' .. token
-    headers['Content-Type'] = 'application/json'
-
-    -- requesting status
-    session = http_mksession()
-    url = url_join(instance, 'api/states/' .. entity)
-    req = http_request(session, 'GET', url, {
-        headers=headers
-    })
-    r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then
-        return 'http error: ' .. r['status']
-    end
-
-    m = json_decode(r['text'])
-    if last_err() then return end
-    debug(m)
-
-    info({
-        gps_accuracy=m['attributes']['gps_accuracy'],
-        longitude=m['attributes']['longitude'],
-        latitude=m['attributes']['latitude'],
-        last_updated=m['last_updated'],
-    })
-end

modules/dev/images.lua

@@ -1,9 +0,0 @@
--- Description: Parse image metadata
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: images
-
-function run(arg)
-    local img = img_load(arg['value'])
-    db_update('image', arg, img)
-end

modules/dev/instagram.lua

@@ -1,150 +0,0 @@
--- Description: Collect data from instagram profiles
--- Version: 0.2.0
--- Source: accounts:instagram.com
--- License: GPL-3.0
-
-PAGE_SIZE = 50
-
-function get_shared_data(html)
-    local s = html_select_list(html, 'script')
-
-    for i=1, #s do
-        local m = regex_find('^window\\._sharedData = (.+);$', s[i]['text'])
-        if m then
-            return json_decode(m[2])
-        end
-    end
-end
-
-function sign_request(rhx, json_params)
-    local magic = rhx .. ':' .. json_params
-    local x_instagram_gis = hex(md5(magic))
-    return x_instagram_gis
-end
-
-function download_image(node)
-    local url = node['display_url']
-    debug(url)
-
-    local req = http_request(session, 'GET', url, {
-        into_blob=true,
-    })
-    local r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-
-    db_add('image', {
-        value=r['blob'],
-    })
-end
-
-function pull_graphql(page)
-    local end_cursor = page['page_info']['end_cursor']
-
-    for i=1, #page['edges'] do
-        -- shortcode = page['edges'][i]['shortcode']
-        local node = page['edges'][i]['node']
-        node['thumbnail_resources'] = nil
-        node['media_preview'] = nil
-        -- debug(node)
-
-        -- if node['__typename'] == 'GraphImage'
-
-        -- node['dimensions']['height']
-        -- node['dimensions']['width']
-        -- ^ not sure how to get that picture
-
-        -- node['taken_at_timestamp']
-        -- location = node['location']
-
-        local err = download_image(node)
-        if last_err() then return end
-        if err ~= nil then return err end
-
-        todo_posts = todo_posts -1
-        debug('posts left: ' .. todo_posts .. '/' .. total_posts)
-    end
-
-    if page['page_info']['has_next_page'] then
-        debug('requesting next page=' .. end_cursor)
-
-        variables = json_encode({
-            id=user['id'],
-            first=PAGE_SIZE,
-            after=end_cursor
-        })
-
-        local headers = {}
-        headers['X-Instagram-GIS'] = sign_request(rhx_gis, variables)
-
-        local req = http_request(session, 'GET', 'https://www.instagram.com/graphql/query/', {
-            query={
-                query_hash='42323d64886122307be10013ad2dcc44',
-                variables=variables,
-            },
-            headers=headers,
-        })
-        r = http_send(req)
-        if last_err() then return end
-        if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-
-        x = json_decode(r['text'])
-        if last_err() then return end
-        return pull_graphql(x['data']['user']['edge_owner_to_timeline_media'])
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    local url = 'https://www.instagram.com/' .. arg['username'] .. '/'
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'invalid status code' end
-    local html = resp['text']
-
-    local ld = html_select(html, 'script[type="application/ld+json"]')
-    if last_err() then return end
-
-    local ld = json_decode(ld['text'])
-    if last_err() then return end
-    --debug(ld)
-
-    if ld['email'] then
-        db_add('email', {
-            value=ld['email'],
-        })
-    end
-
-    -- homepage=ld['url']
-
-    db_update('account', arg, {
-        displayname=ld['name'],
-        email=ld['email'],
-        url=url,
-    })
-
-    -- download images
-    local sd = get_shared_data(html)
-    if last_err() then return end
-    -- debug(sd)
-
-    rhx_gis = sd['rhx_gis']
-    user = sd['entry_data']['ProfilePage'][1]['graphql']['user']
-
-    -- user['full_name']
-    -- user['id']
-    -- user['is_business_account']
-    -- user['is_private']
-    -- user['is_verified']
-    -- user['has_blocked_viewer']
-    -- user['connected_fb_page']
-    -- user['country_block']
-
-    local page = user['edge_owner_to_timeline_media']
-    total_posts = page['count']
-    todo_posts = total_posts
-
-    -- TODO: fast-update abort if image has been downloaded already
-    return pull_graphql(page)
-end

modules/dev/isc-dhcpd-leases.lua

@@ -1,78 +0,0 @@
--- Description: Parse isc-dhcpd dhcpd.leases(5)
--- Version: 0.2.0
--- License: GPL-3.0
-
--- cat /var/lib/dhcpd/dhcpd.leases
-
-function add(lease)
-    if not lease['active'] then return end
-
-    now = datetime()
-
-    device_id = db_add('device', {
-        value=lease['mac'],
-        hostname=lease['hostname'],
-        last_seen=now,
-    })
-    if last_err() then return end
-
-    db_add_ttl('network-device', {
-        network_id=network_id,
-        device_id=device_id,
-        ipaddr=lease['ipaddr'],
-        last_seen=now,
-    }, 180)
-    if last_err() then return end
-end
-
-function each_line(x)
-    debug(x)
-    m = regex_find('^lease (\\S+) \\{\n$', x)
-    if m then
-        lease = {}
-        debug('ipaddr=' .. m[2])
-        lease['ipaddr'] = m[2]
-    end
-    m = regex_find('^\\s*hardware ethernet (\\S+);\n$', x)
-    if m then
-        debug('mac=' .. m[2])
-        lease['mac'] = m[2]
-    end
-    m = regex_find('^\\s*client-hostname \"(.+)\";\n$', x)
-    if m then
-        debug('hostname=' .. m[2])
-        lease['hostname'] = m[2]
-    end
-    m = regex_find('^\\s*binding state active;\n$', x)
-    if m then
-        debug('active=true')
-        lease['active'] = true
-    end
-    m = regex_find('^\\}\n$', x)
-    if m then
-        add(lease)
-    end
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        if not regex_find('^\\s*(#.*|\\s*)\n$', x) then
-            each_line(x)
-        end
-    end
-end

modules/dev/iw-station-dump.lua

@@ -1,77 +0,0 @@
--- Description: Parse iw station dump
--- Version: 0.2.0
--- License: GPL-3.0
-
--- iw dev wlan0 station dump
-
-function add(client)
-    if
-        client['authenticated'] == 'yes' and
-        client['authorized'] == 'yes' and
-        client['mac']
-    then
-        debug(client)
-
-        now = datetime()
-
-        device_id = db_add('device', {
-            value=client['mac'],
-            last_seen=now,
-        })
-        if last_err() then return end
-
-        db_add_ttl('network-device', {
-            network_id=network_id,
-            device_id=device_id,
-            last_seen=now,
-        }, 180)
-        if last_err() then return end
-    end
-
-    client = nil
-end
-
-function each_line(x)
-    debug(x)
-    m = regex_find('^Station (\\S+)', x)
-    if m then
-        if client then
-            add(client)
-        end
-        client = {}
-        client['mac'] = m[2]
-        debug('mac=' .. m[2])
-    end
-
-    m = regex_find('^\\s+([^:]+):\\s*(.+)\n$', x)
-    if m and client then
-        client[m[2]] = m[3]
-        debug(m[2] .. '=' .. m[3])
-    end
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    client = nil
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        each_line(x)
-    end
-
-    if client then
-        add(client)
-    end
-end

modules/dev/keybase-domains.lua

@@ -1,28 +0,0 @@
--- Description: Find keybase proofs for domains
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: domains
-
-function run(arg)
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
-        query={
-            domain=arg['value'],
-        }
-    })
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    x = json_decode(resp['text'])
-    if last_err() then return end
-    debug(x)
-
-    if x['them'][1] == nil then return end
-    them = x['them'][1]
-
-    db_add('account', {
-        service='keybase.io',
-        username=them['basics']['username'],
-    })
-end

modules/dev/keybase-profiles.lua

@@ -1,44 +0,0 @@
--- Description: Find keybase proofs for online accounts
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: accounts
-
-function run(arg)
-    service = arg['service']
-    if service == 'twitter.com' then
-        service = 'twitter'
-    elseif service == 'github.com' then
-        service = 'github'
-    elseif service == 'reddit.com' then
-        service = 'reddit'
-    elseif service == 'news.ycombinator.com' then
-        service = 'hackernews'
-    elseif service == 'facebook.com' then
-        service = 'facebook'
-    else
-        return
-    end
-
-    query = {}
-    query[service] = arg['username']
-
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
-        query=query,
-    })
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    x = json_decode(resp['text'])
-    if last_err() then return end
-    debug(x)
-
-    if x['them'][1] == nil then return end
-    them = x['them'][1]
-
-    db_add('account', {
-        service='keybase.io',
-        username=them['basics']['username'],
-    })
-end

modules/dev/keybase.lua

@@ -1,85 +0,0 @@
--- Description: Collect accounts and emails from keybase accounts
--- Version: 0.2.0
--- License: GPL-3.0
--- Source: accounts:keybase.io
-
-function extract_mails(pubkey)
-    for j=1, #pubkey['uids'] do
-        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
-        if m then
-            db_add('email', {
-                value=m[3],
-                displayname=m[2],
-            })
-        end
-    end
-end
-
-function add_domain(dns)
-    local domain = psl_domain_from_dns_name(dns)
-    if last_err() then return end
-
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    if domain ~= dns then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=dns,
-        })
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
-        query={
-            usernames=arg['username'],
-        }
-    })
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    x = json_decode(resp['text'])
-    if last_err() then return end
-    debug(x)
-
-    if x['them'][1] == nil then return end
-    them = x['them'][1]
-
-    -- update keybase profile
-    db_update('account', arg, {
-        displayname=them['profile']['full_name'],
-        url='https://keybase.io/'..arg['username'],
-    })
-
-    -- collect emails
-    pubkey = pgp_pubkey_armored(them['public_keys']['primary']['bundle'])
-    debug(pubkey)
-    extract_mails(pubkey)
-
-    -- collect profiles
-    profiles = them['proofs_summary']['all']
-
-    for i=1, #profiles do
-        profile = profiles[i]
-        debug(profile)
-
-        if
-            profile['proof_type'] == 'generic_web_site' or
-            profile['proof_type'] == 'dns'
-        then
-            add_domain(profile['nametag'])
-        else
-            db_add('account', {
-                service=profile['proof_type'],
-                username=profile['nametag'],
-                url=profile['service_url'],
-            })
-        end
-    end
-end

modules/dev/namechk.lua

@@ -1,85 +0,0 @@
--- Description: Find accounts by username with namechk.com
--- Version: 0.2.0
--- Source: accounts
--- License: GPL-3.0
-
-function get_services(html)
-    local divs = html_select_list(html, '.service')
-    if last_err() then return end
-
-    local services = {}
-
-    for i=1, #divs do
-        services[i] = divs[i]['attrs']['data-name']
-    end
-
-    return services
-end
-
-function run(arg)
-    -- setup session
-    local session = http_mksession()
-    local req = http_request(session, 'GET', 'https://namechk.com/', {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    local token = html_select(resp['text'], 'input[name="authenticity_token"]')
-    local auth_token = token['attrs']['value']
-
-        local headers = {}
-        headers['X-CSRF-Token'] = authenticity_token
-
-    local services = get_services(resp['text'])
-    debug({
-        auth_token=auth_token,
-        services=services,
-    })
-
-    -- trigger the scan
-    local req = http_request(session, 'POST', 'https://namechk.com/', {
-            headers=headers,
-        form={
-            authenticity_token=auth_token,
-            q=arg['username'],
-        }
-    })
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-    debug(resp)
-
-    local scan = json_decode(resp['text'])
-    if last_err() then return end
-    local scan_token = scan['valid']
-
-    -- get results
-    for i=1, #services do
-        debug(services[i])
-
-        local req = http_request(session, 'POST', 'https://namechk.com/services/check', {
-            headers=headers,
-            form={
-                token=scan_token,
-                fat=auth_token,
-                service=services[i],
-            }
-        })
-        local resp = http_send(req)
-        if last_err() then return end
-
-        if resp['status'] == 200 then
-            local acc = json_decode(resp['text'])
-            if last_err() then return end
-            debug(acc)
-
-            if acc ~= nil and not acc['available'] and acc['status'] == 'unavailable' then
-                db_add('account', {
-                    service=services[i],
-                    username=arg['username'],
-                    url=acc['callback_url'],
-                })
-            end
-        end
-    end
-end

modules/dev/nudity.lua

@@ -1,12 +0,0 @@
--- Description: Scan collected images for nudity
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: images
-
-function run(arg)
-    local nudity = img_nudity(arg['value'])
-    debug(nudity)
-    db_update('image', arg, {
-        nudity=nudity['score'],
-    })
-end

modules/dev/otx-subdomains.lua

@@ -1,29 +0,0 @@
--- Description: Query alienvault otx passive dns for subdomains of a domain
--- Version: 0.3.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    url = 'https://otx.alienvault.com/api/v1/indicators/domain/' .. arg['value'] .. '/passive_dns'
-
-    req = http_request(session, 'GET', url, {})
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['passive_dns']
-
-    for i=1, #o do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x['hostname'],
-        })
-    end
-end

modules/dev/passive-arp.lua

@@ -1,79 +0,0 @@
--- Description: Passive arp-scanner with sniffglue
--- Version: 0.1.0
--- License: GPL-3.0
-
--- sudo sniffglue -jv enp0s25
-
-function each_frame(frame)
-    if not frame['Ether'] then return end
-
-    local arp = frame['Ether'][2]['Arp']
-    if not arp then return end
-
-    if arp['Request'] then
-        arp = arp['Request']
-    elseif arp['Reply'] then
-        arp = arp['Reply']
-    else
-        -- unknown, abort
-        return
-    end
-
-    debug(arp)
-
-    -- TODO: this might change to a string in the future
-    local mac = mac(arp['src_mac'])
-    local ipaddr = arp['src_addr']
-    debug({src_mac=mac, src_addr=ipaddr})
-
-    local now = datetime()
-
-    local device_id = db_add('device', {
-        value=mac,
-        last_seen=now,
-    })
-    if last_err() then return end
-
-    db_add_ttl('network-device', {
-        network_id=network_id,
-        device_id=device_id,
-        ipaddr=ipaddr,
-        last_seen=now,
-    }, 120)
-    if last_err() then return end
-end
-
-function mac(m)
-    return
-        hex({m[1]}) .. ':' ..
-        hex({m[2]}) .. ':' ..
-        hex({m[3]}) .. ':' ..
-        hex({m[4]}) .. ':' ..
-        hex({m[5]}) .. ':' ..
-        hex({m[6]})
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        local x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        local frame = json_decode(x)
-        if last_err() then return end
-
-        each_frame(frame)
-        if last_err() then return end
-    end
-end

modules/dev/passive-spider.lua

@@ -1,60 +0,0 @@
--- Description: Scrape known http responses for urls
--- Version: 0.2.0
--- Source: urls
--- License: GPL-3.0
-
-function entry(parent, href)
-    -- TODO: parse mailto:foo@example.com?subject=asdf
-    -- TODO: parse tel:+4912345
-    -- TODO: allow discovering 3rd-party domains
-    -- TODO: maybe record urls as well
-
-    local psl, parts, url, host
-
-    if href == nil then
-        return
-    end
-
-    url = url_join(parent, href)
-    if last_err() then return clear_err() end
-    if url:match('^https?://') == nil then
-        return
-    end
-
-    parts = url_parse(url)
-    if last_err() then return end
-    host = parts['host']
-    psl = psl_domain_from_dns_name(host)
-
-
-    domain_id = db_select('domain', psl)
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=host,
-        })
-    end
-end
-
-function run(arg)
-    if arg['body'] == nil or #arg['body'] == 0 then
-        return
-    end
-
-    body = utf8_decode(arg['body'])
-    if last_err() then return end
-
-    links = html_select_list(body, 'a')
-    if last_err() then return end
-
-    if #links == 0 then
-        return
-    end
-
-    -- process html links
-    for i=1, #links do
-        href = links[i]['attrs']['href']
-
-        entry(arg['value'], href)
-    end
-end

modules/dev/pgp-keyserver.lua

@@ -1,54 +0,0 @@
--- Description: Query pgp keyserver for email addresses
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    --lookup_url = 'https://pgp.mit.edu/pks/lookup'
-    lookup_url = 'https://sks-keyservers.net/pks/lookup'
-
-    req = http_request(session, 'GET', lookup_url, {
-        query={
-            search=arg['value'],
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    links = html_select_list(resp['text'], 'a')
-    for i=1, #links do
-        href = links[i]['attrs']['href']
-
-        if href:find('/pks/lookup%?op=get&search=') == 1 then
-            url = url_join(lookup_url, href)
-
-            req = http_request(session, 'GET', url, {})
-
-            resp = http_send(req)
-            -- TODO: do not abort script if one attempt fails
-            if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-            pubkey = pgp_pubkey_armored(resp['text'])
-
-            -- print(pubkey)
-
-            -- TODO: ensure at least one email matches our target domain
-            if pubkey['uids'] then
-                for j=1, #pubkey['uids'] do
-                    local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
-                    if m then
-                        db_add('email', {
-                            value=m[3],
-                            displayname=m[2],
-                        })
-                    end
-                end
-            end
-        end
-    end
-end

modules/dev/phpmyadmin.lua

@@ -1,103 +0,0 @@
--- Description: Search for phpmyadmin
--- Version: 0.2.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    paths = {
-        "phpmyadmin/index.php",
-        "phpMyAdmin/index.php",
-        "pmd/index.php",
-        "pma/index.php",
-        "PMA/index.php",
-        "PMA2/index.php",
-        "pmamy/index.php",
-        "pmamy2/index.php",
-        "mysql/index.php",
-        "admin/index.php",
-        "db/index.php",
-        "dbadmin/index.php",
-        "web/phpMyAdmin/index.php",
-        "admin/pma/index.php",
-        "admin/PMA/index.php",
-        "admin/mysql/index.php",
-        "admin/mysql2/index.php",
-        "admin/phpmyadmin/index.php",
-        "admin/phpMyAdmin/index.php",
-        "admin/phpmyadmin2/index.php",
-        "mysqladmin/index.php",
-        "mysql-admin/index.php",
-        "mysql_admin/index.php",
-        "phpadmin/index.php",
-        "phpAdmin/index.php",
-        "phpmyadmin0/index.php",
-        "phpmyadmin1/index.php",
-        "phpmyadmin2/index.php",
-        "phpMyAdmin-4.4.0/index.php",
-        "myadmin/index.php",
-        "myadmin2/index.php",
-        "xampp/phpmyadmin/index.php",
-        "phpMyadmin_bak/index.php",
-        "www/phpMyAdmin/index.php",
-        "tools/phpMyAdmin/index.php",
-        "phpmyadmin-old/index.php",
-        "phpMyAdminold/index.php",
-        "phpMyAdmin.old/index.php",
-        "pma-old/index.php",
-        "claroline/phpMyAdmin/index.php",
-        "typo3/phpmyadmin/index.php",
-        "phpma/index.php",
-        "phpmyadmin/phpmyadmin/index.php",
-        "phpMyAdmin/phpMyAdmin/index.php",
-        "phpMyAbmin/index.php",
-        "phpMyAdmin__/index.php",
-        "phpMyAdmin+++---/index.php",
-        "v/index.php",
-        "phpmyadm1n/index.php",
-        "phpMyAdm1n/index.php",
-        "shaAdmin/index.php",
-        "phpMyadmi/index.php",
-        "phpMyAdmion/index.php",
-        "MyAdmin/index.php",
-        "phpMyAdmin1/index.php",
-        "phpMyAdmin123/index.php",
-        "pwd/index.php",
-        "phpMyAdmina/index.php",
-        "program/index.php",
-        "shopdb/index.php",
-        "phppma/index.php",
-        "phpmy/index.php",
-        "mysql/admin/index.php",
-        "mysql/dbadmin/index.php",
-        "mysql/sqlmanager/index.php",
-        "mysql/mysqlmanager/index.php",
-        "wp-content/plugins/portable-phpmyadmin/wp-pma-mod/index.php",
-    }
-
-    session = http_mksession()
-
-    for i=1, #paths do
-        p = paths[i]
-        url = url_join(arg['value'], p)
-        debug(url)
-
-        req = http_request(session, 'GET', url, {
-            timeout=5000
-        })
-        reply = http_send(req)
-        debug(reply)
-
-        if last_err() then
-            clear_err()
-        else
-            if reply['status'] == 200 then
-                db_add('url', {
-                    subdomain_id=arg['subdomain_id'],
-                    value=url,
-                    status=reply['status'],
-                    body=reply['text'],
-                })
-            end
-        end
-    end
-end

modules/dev/smtp-check.lua

@@ -1,88 +0,0 @@
--- Description: Verify email address by asking the smtp server
--- Version: 0.2.0
--- Source: emails
--- License: GPL-3.0
-
-function find_mx(domain)
-    local records, i, r
-
-    records = dns(domain, {
-        record='MX',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-    -- debug(records)
-
-    for i=1, #records do
-        r = records[i][2]['MX']
-        if r then
-            debug('mx: ' .. r[2])
-            return r[2]
-        end
-    end
-end
-
-function run(arg)
-    -- extract domain
-    domain = arg['value']:match('@(.*)')
-    if doman ~= nil then
-        -- malformed domain
-        return
-    end
-
-    -- mx lookup
-    mx = find_mx(domain)
-    if last_err() then return end
-    if not mx then return end
-
-    -- create connection
-    c = sock_connect(mx, 25, {})
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- send hello
-    sock_sendline(c, 'ehlo localhost')
-    if last_err() then return end
-
-    l = sock_recvline_regex(c, '^250 ')
-    if last_err() then return end
-    debug(l)
-
-    -- send email
-    sock_sendline(c, 'mail from:<root@localhost>')
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- send rcpt
-    sock_sendline(c, 'rcpt to:<' .. arg['value'] .. '>')
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- check status
-    verified = nil
-    if l:match('^2') then
-        debug('email is valid')
-        verified = true
-    elseif l:match('^5') then
-        debug('email is invalid')
-        verified = false
-    elseif l:match('^4') then
-        debug('unknown status, temporary delivery failure')
-    end
-
-    if verified ~= nil then
-        db_update('email', arg, {
-            valid=verified,
-        })
-    end
-end

modules/dev/threatminer-ipaddr.lua

@@ -1,49 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of an ip address
--- Version: 0.3.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- TODO: add option to filter old entries based on last_seen
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/host.php', {
-        query={
-            rt='2',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    for i=1, #o do
-        x = o[i]
-
-        domain = psl_domain_from_dns_name(x['domain'])
-        -- TODO: if this fails, skip this entry instead
-        if last_err() then return end
-
-        domain_id = db_add('domain', {
-            value=domain,
-        })
-
-        if domain_id ~= nil then
-            subdomain_id = db_add('subdomain', {
-                domain_id=domain_id,
-                value=x['domain'],
-            })
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=subdomain_id,
-                ip_addr_id=arg['id'],
-            })
-        end
-    end
-end

modules/dev/threatminer-subdomains.lua

@@ -1,32 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of a domain
--- Version: 0.3.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/domain.php', {
-        query={
-            rt='5',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    for i=1, #o do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x,
-        })
-    end
-end

modules/dev/thunderbird-autoconfig.lua

@@ -1,45 +0,0 @@
--- Description: Query thunderbird autoconfig db for subdomains
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- check if an autoconfig exists without disclosing our target yet
-    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/', {})
-    resp = http_send(req)
-    if last_err() then return end
-
-    if resp['status'] ~= 200 then
-        return 'index request failed'
-    end
-
-    if resp['text']:find(arg['value'], 1, true) == nil then
-        debug('no autoconfig available')
-        return
-    end
-
-    -- request config
-    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/' .. arg['value'], {})
-    resp = http_send(req)
-    if last_err() then return end
-
-    m = regex_find_all('<hostname>([^<]+)</hostname>', resp['text'])
-
-    for i=1, #m do
-        subdomain = m[i][2]
-
-        domain = psl_domain_from_dns_name(subdomain)
-        if last_err() then return end
-
-        domain_id = db_select('domain', domain)
-        if last_err() then return end
-
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=subdomain,
-        })
-        if last_err() then return end
-    end
-end

modules/dev/tld-scan.lua

@@ -1,46 +0,0 @@
--- Description: Search for the same domain base on all TLDs
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: domains
-
-function run(arg)
-    local m = regex_find('^([^\\.]+)\\.', arg['value'])
-    local base = m[2]
-
-    -- TODO: we need a way to cache this
-    -- TODO: .co.uk is missing
-    local url = 'https://data.iana.org/TLD/tlds-alpha-by-domain.txt'
-
-    local session = http_mksession()
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then
-        return 'http error: ' .. resp['status']
-    end
-
-    local tlds = regex_find_all('([^\n]+)', resp['text'])
-
-    for i=1, #tlds do
-        local tld = tlds[i][1]:lower()
-
-        if not tld:match('^#') then
-            local domain = base .. '.' .. tld
-
-            debug(domain)
-            records = dns(domain, {
-                record='NS',
-            })
-            if last_err() then
-                clear_err()
-            else
-                if records['error'] == nil and records['answers'][1] then
-                    debug(records)
-                    db_add('domain', {
-                        value=domain,
-                    })
-                end
-            end
-        end
-    end
-end

modules/dev/twilio-lookup.lua

@@ -1,48 +0,0 @@
--- Description: Retrieve additional information about a phone number
--- Version: 0.1.0
--- Source: phonenumbers
--- Keyring-Access: twilio
--- License: GPL-3.0
-
-function run(arg)
-    number = url_escape(arg['value'])
-    --url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number
-    url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number .. '?Type=carrier&Type=caller-name'
-
-    --debug(url)
-
-    key = keyring('twilio')[1]
-    if not key then
-        return 'Missing required twilio access key'
-    end
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {
-        basic_auth={key['access_key'], key['secret_key']},
-    })
-    reply = http_send(req)
-    if last_err() then return end
-
-    if reply['status'] ~= 200 then
-        return 'api returned error'
-    end
-
-    v = json_decode(reply['text'])
-    if last_err() then return end
-    debug(v)
-
-    update = {}
-    update['country'] = v['country_code']
-
-    if v['carrier'] then
-        update['carrier'] = v['carrier']['name']
-        update['line'] = v['carrier']['type']
-    end
-
-    if v['caller_name'] then
-        update['caller_name'] = v['caller_name']['caller_name']
-        update['caller_type'] = v['caller_name']['caller_type']
-    end
-
-    db_update('phonenumber', arg, update)
-end

modules/dev/url-scan.lua

@@ -1,45 +0,0 @@
--- Description: Scan subdomains for websites
--- Version: 0.3.0
--- Source: subdomains
--- License: GPL-3.0
-
-function request(subdomain_id, url)
-    req = http_request(session, 'GET', url, {
-        timeout=5000
-    })
-    reply = http_send(req)
-
-    if last_err() then
-        clear_err()
-        return
-    end
-
-    obj = {
-        subdomain_id=subdomain_id,
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-    }
-
-    redirect = reply['headers']['location']
-    if redirect then
-        obj['redirect'] = url_join(url, redirect)
-    end
-
-    db_add('url', obj)
-
-    -- debug(reply['status'])
-    -- debug(reply['headers']['location'])
-    -- debug(reply['text'])
-end
-
-function run(arg)
-    domain = arg['value']
-
-    session = http_mksession()
-
-    request(arg['id'], 'http://' .. domain .. '/')
-    if last_err() then return end
-    request(arg['id'], 'https://' .. domain .. '/')
-    if last_err() then return end
-end

modules/dev/waybackurls.lua

@@ -1,54 +0,0 @@
--- Description: Discover subdomains from wayback machine
--- Version: 0.4.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-
-    domain = arg['value']
-    url = 'https://web.archive.org/cdx/search/cdx?url=*.' .. domain .. '/*&output=json&collapse=urlkey'
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-
-    -- no known urls
-    if o[1] == nil then
-        return
-    end
-
-    -- ensure the api response is still what we expect
-    if o[1][3] == nil then
-        return 'api returned unexpected json format'
-    end
-
-    seen = {}
-
-    for i=2, #o do
-        url = o[i][3]
-        debug(url)
-        parts = url_parse(url)
-
-        if last_err() then
-            clear_err()
-            error("Failed to parse url: " .. json_encode(url))
-        else
-            subdomain = parts['host']
-            subdomain, _ = subdomain:gsub('%.$', '')
-
-            if seen[subdomain] == nil then
-                db_add('subdomain', {
-                    domain_id=arg['id'],
-                    value=parts['host'],
-                })
-                if last_err() then return end
-                seen[subdomain] = 1
-            end
-        end
-    end
-end

modules/dev/well-known-uris.lua

@@ -1,61 +0,0 @@
--- Description: Scan for known /.well-known/ locations
--- Version: 0.2.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    -- https://www.iana.org/assignments/well-known-uris/well-known-uris.xhtml
-    -- https://en.wikipedia.org/wiki/List_of_/.well-known/_services_offered_by_webservers
-
-    -- TODO: check if every location causes a 200/redirect
-
-    locations = {
-        {path='security.txt'}, -- expect 200
-        {path='dnt-policy.txt'}, -- expect 200
-        {path='caldav', redirect=true}, -- expect redirect
-        {path='autoconfig/mail/config-v1.1.xml'}, -- expect 200
-        {path='assetlinks.json'}, -- expect 200
-        {path='apple-app-site-association'},  -- expect 200
-        {path='keybase.txt'}, -- expect 200
-        {path='apple-developer-merchantid-domain-association'}, -- expect 200
-        {path='openpgpkey'}, -- expect 200
-        {path='change-password', redirect=true}, -- expect redirect
-    }
-
-    session = http_mksession()
-
-    for i=1, #locations do
-        path = locations[i]['path']
-        expect_redirect = locations[i]['redirect']
-
-        url = url_join(arg['value'], '/.well-known/' .. path)
-        debug(url)
-
-        req = http_request(session, 'GET', url, {
-            timeout=5000,
-        })
-        reply = http_send(req)
-        debug(reply)
-
-        if last_err() then
-            clear_err()
-        else
-            status = reply['status']
-            if (status == 200 and not expect_redirect) or (expect_redirect and status >= 300 and status < 400) then
-                obj = {
-                    subdomain_id=arg['subdomain_id'],
-                    value=url,
-                    status=reply['status'],
-                    body=reply['text'],
-                }
-
-                redirect = reply['headers']['location']
-                if redirect then
-                    obj['redirect'] = url_join(url, redirect)
-                end
-
-                db_add('url', obj)
-            end
-        end
-    end
-end

modules/harness/add-all.lua

@@ -0,0 +1,107 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('adding domain')
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+    if last_err() then return end
+
+    info('adding subdomain')
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value='example.com',
+    })
+    if last_err() then return end
+
+    info('adding ipaddr')
+    ipaddr_id = db_add('ipaddr', {
+        value='192.0.2.1',
+    })
+    if last_err() then return end
+
+    info('adding device')
+    device_id = db_add('device', {
+        value='ff:ff:ff:ff:ff:ff',
+    })
+    if last_err() then return end
+
+    info('adding network')
+    network_id = db_add('network', {
+        value='myssid',
+    })
+    if last_err() then return end
+
+    info('adding email')
+    email_id = db_add('email', {
+        value='foo@example.com',
+    })
+    if last_err() then return end
+
+    info('adding phonenumber')
+    phonenumber_id = db_add('phonenumber', {
+        value='+4912345678',
+    })
+    if last_err() then return end
+
+    info('adding breach')
+    breach_id = db_add('breach', {
+        value='hack the planet',
+    })
+    if last_err() then return end
+
+    info('adding account')
+    account_id = db_add('account', {
+        service='github.com',
+        username='kpcyrd',
+    })
+    if last_err() then return end
+
+    info('adding image')
+    blob = create_blob('abc')
+    image_id = db_add('image', {
+        value=blob,
+    })
+    if last_err() then return end
+
+    info('adding port')
+    port_id = db_add('port', {
+        ip_addr_id=ipaddr_id,
+        ip_addr='192.0.2.1',
+        port=443,
+        protocol='tcp',
+        status='open',
+    })
+    if last_err() then return end
+
+    info('adding url')
+    url_id = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://www.example.com/a/b',
+        body='<html></html>',
+    })
+    if last_err() then return end
+
+    info('adding breach_email')
+    db_add('breach-email', {
+        breach_id=breach_id,
+        email_id=email_id,
+    })
+    if last_err() then return end
+
+    info('adding network_device')
+    db_add('network-device', {
+        network_id=network_id,
+        device_id=device_id,
+    })
+    if last_err() then return end
+
+    info('adding subdomain_ipaddr')
+    db_add('subdomain-ipaddr', {
+        subdomain_id=subdomain_id,
+        ip_addr_id=ipaddr_id,
+    })
+    if last_err() then return end
+end

modules/harness/import-subdomains.lua

@@ -0,0 +1,29 @@
+-- Description: Import subdomains from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lu
+
+    while true do
+        local line = stdin_readline()
+        if line == nil then
+            break
+        end
+        -- strip newline
+        local m = regex_find('.+', line)
+        if m then
+            local subdomain = m[1]
+
+            local domain = psl_domain_from_dns_name(subdomain)
+            local domain_id = db_add('domain', {
+                value=domain,
+            })
+
+            db_add('subdomain', {
+                domain_id=domain_id,
+                value=subdomain,
+            })
+        end
+    end
+end

modules/harness/onion-http.lua

@@ -0,0 +1,16 @@
+-- Description: Send a request to a hidden service
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local session = http_mksession()
+    local req = http_request(session, 'GET', 'http://expyuzz4wqqyqhjn.onion/', {
+        proxy='127.0.0.1:9050',
+    })
+    local r = http_fetch(req)
+    if last_err() then return end
+
+    local title = html_select(r['text'], 'title')
+    if last_err() then return end
+    info(title['html'])
+end

modules/harness/pgp-stdin.lua

@@ -0,0 +1,9 @@
+-- Description: Decode armored pgp key from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local pubkey = stdin_read_to_end()
+    pubkey = pgp_pubkey_armored(pubkey)
+    info(pubkey)
+end

modules/harness/port.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    ip_addr = '192.168.1.2'
+    ip_addr_id = db_add('ipaddr', {
+        value=ip_addr,
+    })
+    db_add('port', {
+        ip_addr_id=ip_addr_id,
+        ip_addr=ip_addr,
+        protocol='tcp',
+        port=4444,
+        status='open',
+    })
+end

modules/harness/tls-insecure.lua

@@ -0,0 +1,16 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    sock = sock_connect('expired.badssl.com', 443, {})
+    if last_err() then return end
+
+    tls = sock_upgrade_tls(sock, {
+        sni_value='expired.badssl.com',
+        disable_tls_verify=true,
+    })
+    if last_err() then return end
+
+    info(tls)
+end

modules/harness/tls.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    sock = sock_connect('badssl.com', 443, {})
+    if last_err() then return end
+
+    tls = sock_upgrade_tls(sock, {
+        sni_value='badssl.com',
+    })
+    if last_err() then return end
+
+    info(tls)
+
+    info(x509_parse_pem(tls['cert']))
+end

modules/harness/tux-img.lua

@@ -8,11 +8,8 @@ function run()
     req = http_request(session, 'GET', 'https://www.kernel.org/theme/images/logos/tux.png', {
         into_blob=true,
     })
-    r = http_send(req)
+    r = http_fetch(req)
     if last_err() then return end
-    if r['status'] ~= 200 then
-        return 'http error: ' .. r['status']
-    end
 
     debug(r)
     db_add('image', {

modules/harness/warn.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run()
+    warn('testing')
+    warn_once('testing once')
+    warn_once('testing once')
+end

modules/harness/xss.lua

@@ -0,0 +1,8 @@
+-- Description: jaVasCript:/*-/*`/*\`/*'/*"/**/(/* */oNcliCk=alert() )//%0D%0A%0d%0a//</stYle/</titLe/</teXtarEa/</scRipt/--!>\x3csVg/<sVg/oNloAd=alert()//>\x3e
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run()
+    -- TODO your code here
+end

sn0int-registry/Cargo.toml

@@ -8,9 +8,10 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.6.0", path="sn0int-common" }
+sn0int-common = { version="0.8.0", path="sn0int-common" }
 rocket = { version = "0.4", default-features=false }
-rocket_failure = { version = "0.1.2", features = ["with-rocket"] }
+#rocket_failure = { path = "../../rocket_failure" }
+rocket_failure = { version = "0.2" }
 rocket_contrib = { version = "0.4.1", features = ["handlebars_templates"] }
 
 diesel = { version = "1.3", features = ["postgres", "r2d2"] }

sn0int-registry/Dockerfile

@@ -1,12 +1,12 @@
-FROM rust
+FROM rust:buster
 RUN rustup install nightly
 WORKDIR /usr/src/sn0int-registry
 COPY . .
 RUN cargo +nightly build --release --verbose
 RUN strip target/release/sn0int-registry
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libcurl3 libpq5 \
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libcurl4 libpq5 \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int-registry/target/release/sn0int-registry /usr/local/bin/sn0int-registry
 COPY templates /templates

sn0int-registry/assets/style.css

@@ -2,8 +2,10 @@ body {
     background-color: black;
     color: #00aa00;
     font-family: monospace;
-    max-width: 700px;
-    margin: 0 auto;
+    max-width: 900px;
+    margin: 20px auto;
+    font-size: 14px;
+    line-height: 18px;
 }
 
 input {
@@ -29,7 +31,41 @@ a {
     padding: 10px;
 }
 
-.list-unstyled {
-    list-style: none;
-    padding: 0 0 0 20px;
+.center {
+    text-align: center;
+}
+
+.modules {
+    list-style: none;
+    padding: 0;
+    word-break: break-word;
+}
+
+.modules li {
+    padding: 2px 5px;
+    margin: 1px 0;
+    background-color: #222;
+}
+
+.mod-cell {
+    display: inline-block;
+    vertical-align: top;
+}
+
+.mod-name {
+    width: 35%;
+}
+
+.mod-name a {
+    text-decoration: none;
+    color: #00cc00;
+    font-weight: bold;
+}
+
+.mod-version {
+    width: 15%;
+}
+
+.mod-descr {
+    width: 50%;
 }

sn0int-registry/migrations/2019-05-27-000932_source/down.sql

@@ -0,0 +1,19 @@
+ALTER TABLE modules DROP COLUMN source;
+
+CREATE OR REPLACE FUNCTION modules_vector_update() RETURNS TRIGGER AS $$
+BEGIN
+    IF TG_OP = 'INSERT' THEN
+        new.search_vector = to_tsvector('pg_catalog.english',
+            NEW.name || ' ' || NEW.author || ' ' || NEW.description
+        );
+    END IF;
+    IF TG_OP = 'UPDATE' THEN
+        IF NEW.description <> OLD.description THEN
+            new.search_vector = to_tsvector('pg_catalog.english',
+                NEW.name || ' ' || NEW.author || ' ' || NEW.description
+            );
+        END IF;
+    END IF;
+    RETURN NEW;
+END
+$$ LANGUAGE 'plpgsql';

sn0int-registry/migrations/2019-05-27-000932_source/up.sql

@@ -0,0 +1,19 @@
+ALTER TABLE modules ADD COLUMN source VARCHAR;
+
+CREATE OR REPLACE FUNCTION modules_vector_update() RETURNS TRIGGER AS $$
+BEGIN
+    IF TG_OP = 'INSERT' THEN
+        new.search_vector = to_tsvector('pg_catalog.english',
+            NEW.name || ' ' || NEW.author || ' ' || COALESCE(NEW.source, '') || ' ' || NEW.description
+        );
+    END IF;
+    IF TG_OP = 'UPDATE' THEN
+        IF NEW.description <> OLD.description OR NEW.source <> OLD.source THEN
+            new.search_vector = to_tsvector('pg_catalog.english',
+                NEW.name || ' ' || NEW.author || ' ' || COALESCE(NEW.source, '') || ' ' || NEW.description
+            );
+        END IF;
+    END IF;
+    RETURN NEW;
+END
+$$ LANGUAGE 'plpgsql';

sn0int-registry/sn0int-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.6.0"
+version = "0.8.0"
 description = "Common code for sn0int"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -10,7 +10,7 @@ edition = "2018"
 [dependencies]
 serde = "1.0"
 serde_derive = "1.0"
-#rocket_failure = { path = "../../../rocket_failure" }
-rocket_failure = "0.1.1"
+#rocket_failure_errors = { path = "../../../rocket_failure/rocket_failure_errors" }
+rocket_failure_errors = "0.2"
 failure = "0.1"
-nom = "4.0"
+nom = "5.0"

sn0int-registry/sn0int-common/src/api.rs

@@ -1,3 +1,5 @@
+use crate::id::ModuleID;
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct WhoamiResponse {
     pub user: String,
@@ -29,6 +31,13 @@ pub struct ModuleInfoResponse {
     pub name: String,
     pub description: String,
     pub latest: Option<String>,
+    pub redirect: Option<ModuleID>,
+}
+
+impl ModuleInfoResponse {
+    pub fn canonical(&self) -> String {
+        format!("{}/{}", self.author, self.name)
+    }
 }
 
 #[derive(Debug, Serialize, Deserialize)]

sn0int-registry/sn0int-common/src/id.rs

@@ -1,38 +1,40 @@
 use crate::errors::*;
 use nom;
-use nom::types::CompleteStr;
 use serde::{de, Serialize, Serializer, Deserialize, Deserializer};
 use std::fmt;
 use std::result;
 use std::str::FromStr;
 
 
+#[inline(always)]
 fn valid_char(c: char) -> bool {
-    nom::is_alphanumeric(c as u8) || c == '-'
+    nom::character::is_alphanumeric(c as u8) || c == '-'
 }
 
 pub fn valid_name(name: &str) -> Result<()> {
-    if token(CompleteStr(name)).is_ok() {
+    if token(name).is_ok() {
         Ok(())
     } else {
         bail!("String contains invalid character")
     }
 }
 
-named!(module<CompleteStr, ModuleID>, do_parse!(
-    author: token   >>
-    tag!("/")       >>
-    name: token     >>
-    eof!()          >>
-    (
-        ModuleID {
-            author: author.to_string(),
-            name: name.to_string(),
-        }
-    )
-));
+fn module(s: &str) -> nom::IResult<&str, ModuleID> {
+    let (input, (author, _, name)) = nom::sequence::tuple((
+        token,
+        nom::bytes::complete::tag("/"),
+        token,
+    ))(s)?;
+    Ok((input, ModuleID {
+        author: author.to_string(),
+        name: name.to_string(),
+    }))
+}
 
-named!(token<CompleteStr, CompleteStr>, take_while1!(valid_char));
+#[inline]
+fn token(s: &str) -> nom::IResult<&str, &str> {
+    nom::bytes::complete::take_while1(valid_char)(s)
+}
 
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub struct ModuleID {
@@ -50,8 +52,11 @@ impl FromStr for ModuleID {
     type Err = Error;
 
     fn from_str(s: &str) -> Result<ModuleID> {
-        let (_, module) = module(CompleteStr(s))
+        let (trailing, module) = module(s)
             .map_err(|err| format_err!("Failed to parse module id: {:?}", err))?;
+        if !trailing.is_empty() {
+            bail!("Trailing data in module id");
+        }
         Ok(module)
     }
 }

sn0int-registry/sn0int-common/src/lib.rs

@@ -9,7 +9,7 @@ pub mod metadata;
 pub mod id;
 pub use crate::id::*;
 
-pub use rocket_failure::StrictApiResponse as ApiResponse;
+pub use rocket_failure_errors::StrictApiResponse as ApiResponse;
 
 #[cfg(test)]
 mod tests {

sn0int-registry/sn0int-common/src/metadata.rs

@@ -3,7 +3,7 @@ use crate::errors::*;
 use std::str::FromStr;
 
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum EntryType {
     Description,
     Version,
@@ -40,9 +40,32 @@ pub enum Source {
     Accounts(Option<String>),
     Breaches,
     Images,
+    Ports,
+    Netblocks,
     KeyRing(String),
 }
 
+impl Source {
+    pub fn group_as_str(&self) -> &'static str {
+        match self {
+            Source::Domains => "domains",
+            Source::Subdomains => "subdomains",
+            Source::IpAddrs => "ipaddrs",
+            Source::Urls => "urls",
+            Source::Emails => "emails",
+            Source::PhoneNumbers => "phonenumbers",
+            Source::Networks => "networks",
+            Source::Devices => "devices",
+            Source::Accounts(_) => "accounts",
+            Source::Breaches => "breaches",
+            Source::Images => "images",
+            Source::Ports => "ports",
+            Source::Netblocks => "netblocks",
+            Source::KeyRing(_) => "keyring",
+        }
+    }
+}
+
 impl FromStr for Source {
     type Err = Error;
 
@@ -66,6 +89,8 @@ impl FromStr for Source {
             ("accounts", param) => Ok(Source::Accounts(param.map(String::from))),
             ("breaches", None) => Ok(Source::Breaches),
             ("images", None) => Ok(Source::Images),
+            ("ports", None) => Ok(Source::Ports),
+            ("netblocks", None) => Ok(Source::Netblocks),
             ("keyring", Some(param)) => Ok(Source::KeyRing(param.to_string())),
             (x, Some(param)) => bail!("Unknown Source: {:?} ({:?})", x, param),
             (x, None) => bail!("Unknown Source: {:?}", x),

sn0int-registry/src/models.rs

@@ -4,7 +4,9 @@ use diesel::pg::PgConnection;
 use diesel::sql_types::BigInt;
 use diesel_full_text_search::{plainto_tsquery, TsQueryExtensions};
 use crate::schema::*;
+use std::collections::HashMap;
 use std::time::SystemTime;
+use sn0int_common::metadata::Metadata;
 
 
 #[derive(AsChangeset, Serialize, Deserialize, Queryable, Insertable)]
@@ -51,6 +53,7 @@ type AllModuleColumns = (
     modules::description,
     modules::latest,
     modules::featured,
+    modules::source,
 );
 
 pub const ALL_MODULE_COLUMNS: AllModuleColumns = (
@@ -60,6 +63,7 @@ pub const ALL_MODULE_COLUMNS: AllModuleColumns = (
     modules::description,
     modules::latest,
     modules::featured,
+    modules::source,
 );
 
 #[derive(AsChangeset, Identifiable, Queryable, Serialize, PartialEq, Debug)]
@@ -71,6 +75,7 @@ pub struct Module {
     pub description: String,
     pub latest: Option<String>,
     pub featured: bool,
+    pub source: Option<String>,
 }
 
 impl Module {
@@ -99,7 +104,10 @@ impl Module {
                         .map_err(Error::from)
     }
 
-    pub fn update_or_create(author: &str, name: &str, description: &str, connection: &PgConnection) -> Result<Module> {
+    pub fn update_or_create(author: &str, name: &str, metadata: &Metadata, connection: &PgConnection) -> Result<Module> {
+        let description = metadata.description.as_str();
+        let source = metadata.source.as_ref().map(|x| x.group_as_str());
+
         match Self::find_opt(author, name, connection)? {
             Some(module) => diesel::update(modules::table.filter(modules::columns::id.eq(module.id)))
                             .set(modules::columns::description.eq(description))
@@ -111,6 +119,7 @@ impl Module {
                 name,
                 description,
                 latest: None,
+                source,
             }, connection),
         }
     }
@@ -153,13 +162,14 @@ impl Module {
     pub fn search(query: &str, connection: &PgConnection) -> Result<Vec<(Module, i64)>> {
         let q = plainto_tsquery(query);
 
-        let x: Vec<(i32, String, String, String, Option<String>, bool, i64)> = modules::table.select((
+        let x: Vec<(i32, String, String, String, Option<String>, bool, Option<String>, i64)> = modules::table.select((
                 modules::id,
                 modules::author,
                 modules::name,
                 modules::description,
                 modules::latest,
                 modules::featured,
+                modules::source,
                 diesel::dsl::sql::<BigInt>("coalesce(sum(releases.downloads), 0) AS sum"),
             ))
             .left_join(releases::table)
@@ -171,7 +181,7 @@ impl Module {
             ))
             .load(connection)?;
 
-        Ok(x.into_iter().map(|(id, author, name, description, latest, featured, downloads)| (
+        Ok(x.into_iter().map(|(id, author, name, description, latest, featured, source, downloads)| (
             Module {
                 id,
                 author,
@@ -179,6 +189,7 @@ impl Module {
                 description,
                 latest,
                 featured,
+                source,
             },
             downloads,
         )).collect())
@@ -196,6 +207,54 @@ impl Module {
             .map_err(Error::from)
     }
 
+    pub fn start_page(connection: &PgConnection) -> Result<HashMap<String, Vec<Module>>> {
+        let x: Vec<(i32, String, String, String, Option<String>, bool, Option<String>, i64)> = modules::table.select((
+                modules::id,
+                modules::author,
+                modules::name,
+                modules::description,
+                modules::latest,
+                modules::featured,
+                modules::source,
+                diesel::dsl::sql::<BigInt>("coalesce(sum(releases.downloads), 0) AS sum"),
+            ))
+            .left_join(releases::table)
+            .group_by(modules::id)
+            //.filter(q.matches(modules::search_vector))
+            .order((
+                modules::featured.desc(),
+                diesel::dsl::sql::<BigInt>("sum").desc(),
+            ))
+            .load(connection)?;
+
+        let mut categories: HashMap<_, Vec<_>> = HashMap::new();
+
+        for (id, author, name, description, latest, featured, source, _downloads) in x {
+            let module = Module {
+                id,
+                author,
+                name,
+                description,
+                latest,
+                featured,
+                source,
+            };
+
+            let source = match &module.source {
+                Some(source) => source.as_str(),
+                _ => "none",
+            };
+
+            if let Some(cat) = categories.get_mut(source) {
+                cat.push(module);
+            } else {
+                categories.insert(source.to_string(), vec![module]);
+            }
+        }
+
+        Ok(categories)
+    }
+
     pub fn count(connection: &PgConnection) -> Result<i64> {
         use diesel::dsl::*;
 
@@ -213,6 +272,7 @@ pub struct NewModule<'a> {
     name: &'a str,
     description: &'a str,
     latest: Option<&'a str>,
+    source: Option<&'a str>,
 }
 
 #[derive(AsChangeset, Identifiable, Queryable, Associations, Serialize, PartialEq, Debug)]

sn0int-registry/src/routes/api.rs

@@ -70,6 +70,7 @@ pub fn info(author: String, name: String, connection: db::Connection) -> ApiResu
         name: module.name,
         description: module.description,
         latest: module.latest,
+        redirect: None,
     }))
 }
 
@@ -118,7 +119,7 @@ pub fn publish(name: String, upload: Json<PublishRequest>, session: AuthHeader,
         .public_context("Version is invalid")?;
 
     connection.transaction::<_, WebError, _>(|| {
-        let module = Module::update_or_create(&user, &name, &metadata.description, &connection)
+        let module = Module::update_or_create(&user, &name, &metadata, &connection)
             .private_context("Failed to write module metadata")?;
 
         match Release::try_find(module.id, &version, &connection)? {

sn0int-registry/src/routes/assets.rs

@@ -1,15 +1,35 @@
 use crate::assets::{ASSET_REV, FAVICON, STYLE_SHEET};
-use rocket::http::ContentType;
-use rocket::http::Status;
+use crate::db;
+use crate::errors::ApiResult;
+use crate::models::*;
+use rocket::http::{ContentType, Status};
 use rocket::http::hyper::header::{CacheControl, CacheDirective};
 use rocket_contrib::templates::Template;
+use std::cmp::Ordering;
 
 
 #[get("/")]
-pub fn index() -> Template {
-    Template::render("index", hashmap!{
-        "ASSET_REV" => ASSET_REV.as_str(),
-    })
+pub fn index(connection: db::Connection) -> ApiResult<Template> {
+    let asset_rev = ASSET_REV.as_str();
+
+    let mut modules = Module::start_page(&connection)?
+        .into_iter()
+        .map(|(k, v)| (k, v))
+        .collect::<Vec<_>>();
+
+    modules.sort_by(|a, b| {
+        match (a.0.as_str(), b.0.as_str()) {
+            ("none", "none") => Ordering::Equal,
+            ("none", _) => Ordering::Greater,
+            (_, "none") => Ordering::Less,
+            (a, b) => a.cmp(b),
+        }
+    });
+
+    Ok(Template::render("index", json!({
+        "ASSET_REV": asset_rev,
+        "modules": modules,
+    })))
 }
 
 #[derive(Responder)]

sn0int-registry/src/schema.rs

@@ -21,6 +21,7 @@ table! {
         latest -> Nullable<Varchar>,
         search_vector -> Tsvector,
         featured -> Bool,
+        source -> Nullable<Varchar>,
     }
 }
 

sn0int-registry/templates/base.html.hbs

@@ -6,7 +6,16 @@
         <link rel="stylesheet" href="/assets/{{ASSET_REV}}/style.css">
     </head>
     <body>
-        <h1><a href="/">sn0int</a></h1>
+        <h1 class="center"><a href="/">sn0int registry</a></h1>
+        <nav>
+            <p class="center">
+                [ <a href="https://github.com/kpcyrd/sn0int" target="_blank" rel="noopener">github</a> ]
+                [ <a href="https://sn0int.readthedocs.io/" target="_blank" rel="noopener">docs</a> ]
+                [ <a href="https://www.reddit.com/r/sn0int" target="_blank" rel="noopener">reddit</a> ]
+                [ <a href="https://chaos.social/@sn0int" target="_blank" rel="noopener">mastodon</a> ]
+                [ <a href="https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int" target="_blank" rel="noopener">irc</a> ]
+            </p>
+        </nav>
 
         {{~> page}}
     </body>

sn0int-registry/templates/index.html.hbs

@@ -1,35 +1,10 @@
 {{#*inline "page"}}
         <p>
-This is the registry server of sn0int, a semi-automatic OSINT framework and
-package manager.  It was built for IT security professionals and bug hunters to
-gather intelligence about a given target or about yourself. sn0int is
-enumerating attack surface by semi-automatically processing public information
-and mapping the results in a unified format for followup investigations.
-        </p>
-
-        <p>
-Among other things, sn0int is currently able to:
-        </p>
-
-        <ul class="list-unstyled">
-<li>- Harvest subdomains from certificate transparency logs</li>
-<li>- Harvest subdomains from various passive dns logs</li>
-<li>- Sift through subdomain results for publicly accessible websites</li>
-<li>- Harvest emails from pgp keyservers</li>
-<li>- Enrich ip addresses with ASN and geoip info</li>
-<li>- Harvest subdomains from the wayback machine</li>
-<li>- Gather information about phonenumbers</li>
-<li>- Bruteforce interesting urls</li>
-        </ul>
-
-        <p>
-sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
-and is fully opensource.  None of the investigations listed above are hardcoded
-in the source, instead those are provided by modules that are executed in a
-sandbox. You can easily extend sn0int by writing your own modules and share
-them with other users by publishing them to the sn0int registry. This allows
-you to ship updates for your modules on your own since you don't need to send a
-pull request.
+        This is the registry server of sn0int, a semi-automatic OSINT framework and
+        package manager.  It was built for IT security professionals and bug hunters to
+        gather intelligence about a given target or about yourself. sn0int is
+        enumerating attack surface by semi-automatically processing public information
+        and mapping the results in a unified format for followup investigations.
         </p>
 
         <p>
@@ -40,37 +15,21 @@ pull request.
         </p>
 
         <p>
-        You can search for modules with sn0int using:
-        </p>
-        <p class="code"><code>
-        sn0int search ctlogs
-        </code></p>
-
-        <p>
-        To install the module afterwards run:
+        To install a module run:
         </p>
         <p class="code"><code>
         sn0int install kpcyrd/ctlogs
         </code></p>
 
-        <p>
-        If you want to publish your own module, you need to login with github first:
-        </p>
-        <p class="code"><code>
-        sn0int login
-        </code></p>
-
-        <p>
-        Afterwards you can upload your module to the registry (this is going to
-        publish your module as yourgithubuser/example):
-        </p>
-        <p class="code"><code>
-        sn0int publish ./path/to/example.lua
-        </code></p>
-
-        <p>
-        For more information, visit <a href="https://github.com/kpcyrd/sn0int"
-        target="_blank" rel="noopener">sn0int on github</a>.
-        </p>
+        {{#each modules}}
+            <h3>[ {{this.0}} ]</h3>
+            <ul class="modules">
+            {{#each this.1}}
+                <li>
+                    <span class="mod-cell mod-name"><a href="#">{{this.author}}/{{this.name}}</a></span><span class="mod-cell mod-version">{{this.latest}}</span><span class="mod-cell mod-descr">{{this.description}}</span>
+                </li>
+            {{/each}}
+            </ul>
+        {{/each}}
 {{/inline}}
 {{~> base }}

src/args.rs

@@ -6,8 +6,7 @@ use crate::workspaces::Workspace;
 
 
 #[derive(Debug, StructOpt)]
-#[structopt(author = "",
-            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
     /// Select a different workspace instead of the default
     #[structopt(short="w", long="workspace")]
@@ -29,40 +28,55 @@ impl Args {
 #[derive(Debug, StructOpt)]
 pub enum SubCommand {
     /// Run a module directly
-    #[structopt(author="", name="run")]
+    #[structopt(name="run")]
     Run(Run),
     /// For internal use
-    #[structopt(author="", name="sandbox")]
+    #[structopt(name="sandbox")]
     Sandbox(Sandbox),
     /// Login to the registry for publishing
-    #[structopt(author="", name="login")]
+    #[structopt(name="login")]
     Login(Login),
     /// Create a new module
-    #[structopt(author="", name="new")]
+    #[structopt(name="new")]
     New(New),
     /// Publish a script to the registry
-    #[structopt(author="", name="publish")]
+    #[structopt(name="publish")]
     Publish(Publish),
     /// Install a module from the registry
-    #[structopt(author="", name="install")]
+    #[structopt(name="install")]
     Install(Install),
     /// Search in the registry
-    #[structopt(author="", name="search")]
+    #[structopt(name="search")]
     Search(Search),
     /// Insert into the database
-    #[structopt(author="", name="add")]
+    #[structopt(name="add")]
     Add(cmd::add_cmd::Args),
     /// Select from the database
-    #[structopt(author="", name="select")]
+    #[structopt(name="select")]
     Select(cmd::select_cmd::Args),
     /// Delete from the database
-    #[structopt(author="", name="delete")]
+    #[structopt(name="delete")]
     Delete(cmd::delete_cmd::Args),
+    /// Include entities in the scope
+    #[structopt(name="scope")]
+    Scope(cmd::scope_cmd::Args),
+    /// Exclude entities from scope
+    #[structopt(name="noscope")]
+    Noscope(cmd::noscope_cmd::Args),
+    /// Manage workspaces
+    #[structopt(name="workspace")]
+    Workspace(cmd::workspace_cmd::Args),
     /// Verify blob storage for corrupt and dangling blobs
-    #[structopt(author="", name="fsck")]
+    #[structopt(name="fsck")]
     Fsck(cmd::fsck_cmd::Args),
+    /// Export a workspace for external processing
+    #[structopt(name="export")]
+    Export(cmd::export_cmd::Args),
+    /// Run a lua repl
+    #[structopt(name="repl")]
+    Repl,
     /// Generate shell completions
-    #[structopt(author="", name="completions")]
+    #[structopt(name="completions")]
     Completions(Completions),
 }
 
@@ -119,7 +133,7 @@ pub struct New {
 #[derive(Debug, StructOpt)]
 pub struct Publish {
     /// The scripts to publish
-    #[structopt(raw(required = "true"))]
+    #[structopt(required = true)]
     pub paths: Vec<String>,
 }
 
@@ -129,16 +143,21 @@ pub struct Install {
     pub module: ModuleID,
     /// Specify the version, defaults to the latest version
     pub version: Option<String>,
+    #[structopt(short="f", long="force")]
+    pub force: bool,
 }
 
 #[derive(Debug, StructOpt)]
 pub struct Search {
+    /// Only show modules that aren't installed yet
+    #[structopt(long="new")]
+    pub new: bool,
     /// The search query
     pub query: String,
 }
 
 #[derive(Debug, StructOpt)]
 pub struct Completions {
-    #[structopt(raw(possible_values="&Shell::variants()"))]
+    #[structopt(possible_values=&Shell::variants())]
     pub shell: Shell,
 }

src/autonoscope/domain.rs

@@ -0,0 +1,135 @@
+use crate::errors::*;
+use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::models::*;
+use std::convert::TryFrom;
+
+#[derive(Debug, PartialEq)]
+pub struct DomainRule {
+    value: String,
+    fragments: Vec<String>,
+}
+
+impl ToString for DomainRule {
+    fn to_string(&self) -> String {
+        self.value.clone()
+    }
+}
+
+impl TryFrom<&str> for DomainRule {
+    type Error = Error;
+
+    fn try_from(rule: &str) -> Result<DomainRule> {
+        let mut fragments = rule.split('.')
+            .filter(|x| !x.is_empty())
+            .map(String::from)
+            .collect::<Vec<_>>();
+        fragments.reverse();
+
+        Ok(DomainRule {
+            value: rule.to_string(),
+            fragments,
+        })
+    }
+}
+
+impl TryFrom<Autonoscope> for DomainRule {
+    type Error = Error;
+
+    #[inline]
+    fn try_from(rule: Autonoscope) -> Result<DomainRule> {
+        DomainRule::try_from(rule.value.as_str())
+    }
+}
+
+impl AutoRule<NewDomain> for DomainRule {
+    #[inline]
+    fn matches(&self, domain: &NewDomain) -> Result<bool> {
+        self.matches(domain.value.as_str())
+    }
+}
+
+impl AutoRule<NewSubdomain> for DomainRule {
+    #[inline]
+    fn matches(&self, domain: &NewSubdomain) -> Result<bool> {
+        self.matches(domain.value.as_str())
+    }
+}
+
+impl AutoRule<NewUrl> for DomainRule {
+    #[inline]
+    fn matches(&self, url: &NewUrl) -> Result<bool> {
+        let url = url.value.parse::<url::Url>()?;
+        if let Some(domain) = url.domain() {
+            self.matches(domain)
+        } else {
+            Ok(false)
+        }
+    }
+}
+
+impl AutoRule<str> for DomainRule {
+    fn matches(&self, domain: &str) -> Result<bool> {
+        let frags = domain.split('.')
+            .filter(|x| !x.is_empty())
+            .collect::<Vec<_>>();
+
+        if self.fragments.len() > frags.len() {
+            return Ok(false);
+        }
+
+        for (rule, domain) in self.fragments.iter().zip(frags.iter().rev()) {
+            if rule != domain {
+                return Ok(false);
+            }
+        }
+
+        Ok(true)
+    }
+}
+
+impl RulePrecision for DomainRule {
+    #[inline]
+    fn precision(&self) -> usize {
+        self.fragments.len()
+    }
+}
+
+impl IntoRule for DomainRule {
+    fn into_rule(&self) -> (&'static str, String) {
+        ("domain", self.to_string())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::convert::TryFrom;
+
+    #[test]
+    fn test_domain_rule_root() {
+        let rule = DomainRule::try_from(".").unwrap();
+        assert!(rule.matches("example.com").unwrap());
+        assert_eq!(rule.precision(), 0);
+    }
+
+    #[test]
+    fn test_domain_rule_com() {
+        let rule = DomainRule::try_from("com").unwrap();
+        assert!(rule.matches("example.com").unwrap());
+        assert_eq!(rule.precision(), 1);
+    }
+
+    #[test]
+    fn test_domain_rule_equals() {
+        let rule = DomainRule::try_from("example.com").unwrap();
+        assert!(rule.matches("example.com").unwrap());
+        assert_eq!(rule.precision(), 2);
+    }
+
+    #[test]
+    fn test_domain_rule_mismatch() {
+        let rule = DomainRule::try_from("foo.example.com").unwrap();
+        assert!(!rule.matches("example.com").unwrap());
+        assert_eq!(rule.precision(), 3);
+    }
+}

src/autonoscope/ip.rs

@@ -0,0 +1,214 @@
+use crate::errors::*;
+use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::models::*;
+use std::convert::TryFrom;
+use std::net;
+use ipnetwork::IpNetwork;
+
+
+#[derive(Debug, PartialEq)]
+pub struct IpRule {
+    network: IpNetwork,
+}
+
+impl ToString for IpRule {
+    fn to_string(&self) -> String {
+        self.network.to_string()
+    }
+}
+
+impl TryFrom<&str> for IpRule {
+    type Error = Error;
+
+    fn try_from(x: &str) -> Result<IpRule> {
+        let network = x.parse::<IpNetwork>()?;
+        Ok(IpRule {
+            network,
+        })
+    }
+}
+
+impl TryFrom<Autonoscope> for IpRule {
+    type Error = Error;
+
+    fn try_from(x: Autonoscope) -> Result<IpRule> {
+        IpRule::try_from(x.value.as_str())
+    }
+}
+
+impl AutoRule<NewIpAddr> for IpRule {
+    fn matches(&self, ipaddr: &NewIpAddr) -> Result<bool> {
+        self.matches(ipaddr.value.as_str())
+    }
+}
+
+impl AutoRule<NewPort> for IpRule {
+    fn matches(&self, port: &NewPort) -> Result<bool> {
+        let addr = port.value.parse::<net::SocketAddr>()?;
+        self.matches(&addr.ip())
+    }
+}
+
+impl AutoRule<NewNetblock> for IpRule {
+    fn matches(&self, netblock: &NewNetblock) -> Result<bool> {
+        let range = netblock.value.parse::<ipnetwork::IpNetwork>()?;
+
+        if self.network.prefix() <= range.prefix() {
+            Ok(self.network.contains(range.ip()))
+        } else {
+            Ok(false)
+        }
+    }
+}
+
+impl AutoRule<str> for IpRule {
+    fn matches(&self, ipaddr: &str) -> Result<bool> {
+        let ipaddr = ipaddr.parse::<net::IpAddr>()?;
+        self.matches(&ipaddr)
+    }
+}
+
+impl AutoRule<net::IpAddr> for IpRule {
+    fn matches(&self, ipaddr: &net::IpAddr) -> Result<bool> {
+        Ok(self.network.contains(ipaddr.clone()))
+    }
+}
+
+impl RulePrecision for IpRule {
+    fn precision(&self) -> usize {
+        self.network.prefix() as usize
+    }
+}
+
+impl IntoRule for IpRule {
+    fn into_rule(&self) -> (&'static str, String) {
+        ("ip", self.to_string())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::convert::TryFrom;
+
+    #[test]
+    fn test_ip_rule_ipv4_root() {
+        let rule = IpRule::try_from("0.0.0.0/0").unwrap();
+        assert!(rule.matches("127.0.0.1").unwrap());
+        assert_eq!(rule.precision(), 0);
+    }
+
+    #[test]
+    fn test_ip_rule_ipv6_root() {
+        let rule = IpRule::try_from("::/0").unwrap();
+        assert!(rule.matches("::1").unwrap());
+        assert_eq!(rule.precision(), 0);
+    }
+
+    #[test]
+    fn test_ip_rule_ipv4_match() {
+        let rule = IpRule::try_from("192.0.2.0/24").unwrap();
+        assert!(rule.matches("192.0.2.1").unwrap());
+        assert_eq!(rule.precision(), 24);
+    }
+
+    #[test]
+    fn test_ip_rule_ipv4_mismatch() {
+        let rule = IpRule::try_from("192.0.2.0/24").unwrap();
+        assert!(!rule.matches("127.0.0.1").unwrap());
+        assert_eq!(rule.precision(), 24);
+    }
+
+    #[test]
+    fn test_ip_rule_ipv6_match() {
+        let rule = IpRule::try_from("2001:db8::/32").unwrap();
+        assert!(rule.matches("2001:DB8::1").unwrap());
+        assert_eq!(rule.precision(), 32);
+    }
+
+    #[test]
+    fn test_ip_rule_ipv6_mismatch() {
+        let rule = IpRule::try_from("2001:db8::/32").unwrap();
+        assert!(!rule.matches("::1").unwrap());
+        assert_eq!(rule.precision(), 32);
+    }
+
+    #[test]
+    fn test_ip_rule_ipv6_on_ipv4_mismatch() {
+        let rule = IpRule::try_from("192.0.2.0/24").unwrap();
+        assert!(!rule.matches("2001:DB8::1").unwrap());
+        assert_eq!(rule.precision(), 24);
+    }
+
+    #[test]
+    fn test_ip_rule_ipv4_on_ipv6_mismatch() {
+        let rule = IpRule::try_from("2001:db8::/32").unwrap();
+        assert!(!rule.matches("192.0.2.1").unwrap());
+        assert_eq!(rule.precision(), 32);
+    }
+
+    #[test]
+    fn test_ip_rule_netblock_inner() {
+        let rule = IpRule::try_from("192.0.2.0/24").unwrap();
+        assert!(rule.matches(&NewNetblock {
+            family: String::from("4"),
+            value: String::from("192.0.2.128/25"),
+            asn: None,
+            as_org: None,
+            description: None,
+            unscoped: false,
+        }).unwrap());
+    }
+
+    #[test]
+    fn test_ip_rule_netblock_equal() {
+        let rule = IpRule::try_from("192.0.2.0/24").unwrap();
+        assert!(rule.matches(&NewNetblock {
+            family: String::from("4"),
+            value: String::from("192.0.2.0/24"),
+            asn: None,
+            as_org: None,
+            description: None,
+            unscoped: false,
+        }).unwrap());
+    }
+
+    #[test]
+    fn test_ip_rule_netblock_outer1() {
+        let rule = IpRule::try_from("192.0.2.0/24").unwrap();
+        assert!(!rule.matches(&NewNetblock {
+            family: String::from("4"),
+            value: String::from("192.0.2.0/23"),
+            asn: None,
+            as_org: None,
+            description: None,
+            unscoped: false,
+        }).unwrap());
+    }
+
+    #[test]
+    fn test_ip_rule_netblock_outer2() {
+        let rule = IpRule::try_from("192.0.2.0/24").unwrap();
+        assert!(!rule.matches(&NewNetblock {
+            family: String::from("4"),
+            value: String::from("192.0.0.0/22"),
+            asn: None,
+            as_org: None,
+            description: None,
+            unscoped: false,
+        }).unwrap());
+    }
+
+    #[test]
+    fn test_ip_rule_netblock_no_overlap() {
+        let rule = IpRule::try_from("192.0.2.0/24").unwrap();
+        assert!(!rule.matches(&NewNetblock {
+            family: String::from("4"),
+            value: String::from("192.0.3.0/24"),
+            asn: None,
+            as_org: None,
+            description: None,
+            unscoped: false,
+        }).unwrap());
+    }
+}

src/autonoscope/mod.rs

@@ -0,0 +1,360 @@
+use crate::db::DatabaseSock;
+use crate::errors::*;
+use diesel;
+use diesel::prelude::*;
+use crate::schema::*;
+use crate::models::*;
+use std::cmp::Ordering;
+use std::convert::{TryInto, TryFrom};
+use std::str::FromStr;
+
+mod domain;
+pub use self::domain::DomainRule;
+mod ip;
+pub use self::ip::IpRule;
+mod url;
+pub use self::url::UrlRule;
+
+#[derive(Identifiable, Queryable, PartialEq, Debug)]
+#[table_name="autonoscope"]
+pub struct Autonoscope {
+    pub id: i32,
+    pub object: String,
+    pub value: String,
+    pub scoped: bool,
+}
+
+#[derive(Insertable, PartialEq, Debug)]
+#[table_name="autonoscope"]
+pub struct NewAutonoscope {
+    pub object: String,
+    pub value: String,
+    pub scoped: bool,
+}
+
+#[derive(Debug, Default, PartialEq)]
+pub struct RuleSet {
+    domains: Vec<Rule<DomainRule>>,
+    ips: Vec<Rule<IpRule>>,
+    urls: Vec<Rule<UrlRule>>,
+}
+
+#[inline(always)]
+fn sort_precision_desc<T: RulePrecision>(a: &T, b: &T) -> Ordering {
+    a.precision()
+        .cmp(&b.precision())
+        .reverse()
+}
+
+impl RuleSet {
+    pub fn load(db: &DatabaseSock) -> Result<Self> {
+        use crate::schema::autonoscope::dsl::*;
+        let rules = autonoscope.load::<Autonoscope>(db)?;
+
+        let mut set = RuleSet::default();
+        for rule in rules {
+            let is_scoped = rule.scoped;
+            let rule_type = rule.object.parse::<RuleType>()?;
+            match rule_type {
+                RuleType::Domain => set.domains.push(Rule::new(rule.try_into()?, is_scoped)),
+                RuleType::Ip => set.ips.push(Rule::new(rule.try_into()?, is_scoped)),
+                RuleType::Url => set.urls.push(Rule::new(rule.try_into()?, is_scoped)),
+            }
+        }
+        set.sort_rules();
+
+        Ok(set)
+    }
+
+    fn sort_rules(&mut self) {
+        self.domains.sort_by(sort_precision_desc);
+        self.ips.sort_by(sort_precision_desc);
+        self.urls.sort_by(sort_precision_desc);
+    }
+
+    pub fn add_rule(&mut self, db: &DatabaseSock, object: &RuleType, value: &str, scoped: bool) -> Result<()> {
+        self.delete_rule(db, object, value)?;
+
+        match object {
+            RuleType::Domain => {
+                let rule = DomainRule::try_from(value)?;
+                let rule = Rule::new(rule, scoped);
+                self.db_add(db, &rule)?;
+                self.domains.push(rule);
+            },
+            RuleType::Ip => {
+                let rule = IpRule::try_from(value)?;
+                let rule = Rule::new(rule, scoped);
+                self.db_add(db, &rule)?;
+                self.ips.push(rule);
+            },
+            RuleType::Url => {
+                let rule = UrlRule::try_from(value)?;
+                let rule = Rule::new(rule, scoped);
+                self.db_add(db, &rule)?;
+                self.urls.push(rule);
+            },
+        }
+        self.sort_rules();
+
+        Ok(())
+    }
+
+    fn db_add<I: Into<NewAutonoscope>>(&mut self, db: &DatabaseSock, rule: I) -> Result<()> {
+        use crate::schema::autonoscope::dsl::*;
+        diesel::insert_into(autonoscope)
+            .values(rule.into())
+            .execute(db)?;
+        Ok(())
+    }
+
+    pub fn delete_rule(&mut self, db: &DatabaseSock, obj: &RuleType, rule: &str) -> Result<()> {
+        match obj {
+            RuleType::Domain => {
+                self.domains.retain(|x| x.to_string().as_str() != rule);
+                self.db_delete(db, obj, &rule)?;
+            },
+            RuleType::Ip => {
+                self.ips.retain(|x| x.to_string().as_str() != rule);
+                self.db_delete(db, obj, &rule)?;
+            },
+            RuleType::Url => {
+                self.urls.retain(|x| x.to_string().as_str() != rule);
+                self.db_delete(db, obj, &rule)?;
+            },
+        }
+        Ok(())
+    }
+
+    fn db_delete(&mut self, db: &DatabaseSock, obj: &RuleType, rule: &str) -> Result<()> {
+        use crate::schema::autonoscope::dsl::*;
+        diesel::delete(autonoscope
+            .filter(object.eq(obj.as_str()))
+            .filter(value.eq(rule)))
+            .execute(db)?;
+        Ok(())
+    }
+
+    pub fn rules(&self) -> Vec<(&'static str, String, bool)> {
+        let mut rules = Vec::new();
+        Self::push_rules_display(&mut rules, &self.domains);
+        Self::push_rules_display(&mut rules, &self.ips);
+        Self::push_rules_display(&mut rules, &self.urls);
+        rules
+    }
+
+    #[inline]
+    fn push_rules_display<T: IntoRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
+        for rule in rules {
+            let (object, value) = rule.into_rule();
+            output.push((object, value, rule.scoped));
+        }
+    }
+
+    pub fn matches(&self, object: &Insert) -> Result<bool> {
+        let scoped = match object {
+            Insert::Domain(domain) => Self::matches_any(&self.domains, domain)?,
+            Insert::Subdomain(subdomain) => Self::matches_any(&self.domains, subdomain)?,
+            Insert::IpAddr(ip_addr) => Self::matches_any(&self.ips, ip_addr)?,
+            Insert::Url(url) => {
+                if let Some(result) = Self::matches_any(&self.domains, url)? {
+                    Some(result)
+                } else if let Some(result) = Self::matches_any(&self.urls, url)? {
+                    Some(result)
+                } else {
+                    None
+                }
+            },
+            // Insert::Email(email) => unimplemented!(),
+            // Insert::Account(account) => unimplemented!(),
+            Insert::Port(port) => Self::matches_any(&self.ips, port)?,
+            Insert::Netblock(netblock) => Self::matches_any(&self.ips, netblock)?,
+            _ => None,
+        };
+        Ok(scoped.unwrap_or(true))
+    }
+
+    fn matches_any<T1, T2>(rules: &[Rule<T1>], object: &T2) -> Result<Option<bool>>
+        where T1: AutoRule<T2>,
+            T1: IntoRule,
+    {
+        for rule in rules {
+            if rule.matches(object)? {
+                return Ok(Some(rule.scoped));
+            }
+        }
+        Ok(None)
+    }
+}
+
+#[derive(Debug)]
+pub enum RuleType {
+    Domain,
+    Ip,
+    Url,
+}
+
+impl RuleType {
+    fn as_str(&self) -> &'static str {
+        match self {
+            RuleType::Domain => "domain",
+            RuleType::Ip => "ip",
+            RuleType::Url => "url",
+        }
+    }
+
+    pub fn list_all() -> &'static [&'static str] {
+        lazy_static! {
+            static ref RULES: Vec<&'static str> = vec![
+                RuleType::Domain.as_str(),
+                RuleType::Ip.as_str(),
+                RuleType::Url.as_str(),
+            ];
+        }
+
+        RULES.as_ref()
+    }
+}
+
+impl FromStr for RuleType {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<RuleType> {
+        match s {
+            "domain" => Ok(RuleType::Domain),
+            "ip" => Ok(RuleType::Ip),
+            "url" => Ok(RuleType::Url),
+            _ => bail!("unknown rule type"),
+        }
+    }
+}
+
+pub trait AutoRule<T: ?Sized> {
+    fn matches(&self, value: &T) -> Result<bool>;
+}
+
+pub trait RulePrecision {
+    fn precision(&self) -> usize;
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Rule<T: IntoRule> {
+    rule: T,
+    scoped: bool,
+}
+
+impl<T: IntoRule> Rule<T> {
+    pub fn new(rule: T, scoped: bool) -> Rule<T> {
+        Rule {
+            rule,
+            scoped,
+        }
+    }
+}
+
+// TODO: maybe drop this
+use std::ops::Deref;
+impl<T: IntoRule> Deref for Rule<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.rule
+    }
+}
+
+impl<T: IntoRule + RulePrecision> RulePrecision for Rule<T> {
+    fn precision(&self) -> usize {
+        self.rule.precision()
+    }
+}
+
+pub trait IntoRule {
+    fn into_rule(&self) -> (&'static str, String);
+}
+
+impl<T: IntoRule> Into<NewAutonoscope> for &Rule<T> {
+    fn into(self) -> NewAutonoscope {
+        let (object, value) = self.rule.into_rule();
+        NewAutonoscope {
+            object: object.to_string(),
+            value,
+            scoped: self.scoped,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::convert::TryFrom;
+
+    #[test]
+    fn test_rule_sort_domains() {
+        let mut set = RuleSet {
+            domains: vec![
+                Rule::new(DomainRule::try_from("com").unwrap(), true),
+                Rule::new(DomainRule::try_from(".").unwrap(), true),
+                Rule::new(DomainRule::try_from("example.com").unwrap(), true),
+            ],
+            ips: vec![],
+            urls: vec![],
+        };
+        set.sort_rules();
+        assert_eq!(set, RuleSet {
+            domains: vec![
+                Rule::new(DomainRule::try_from("example.com").unwrap(), true),
+                Rule::new(DomainRule::try_from("com").unwrap(), true),
+                Rule::new(DomainRule::try_from(".").unwrap(), true),
+            ],
+            ips: vec![],
+            urls: vec![],
+        });
+    }
+
+    #[test]
+    fn test_rule_sort_ips() {
+        let mut set = RuleSet {
+            domains: vec![],
+            ips: vec![
+                Rule::new(IpRule::try_from("10.0.0.0/8").unwrap(), true),
+                Rule::new(IpRule::try_from("0.0.0.0/0").unwrap(), true),
+                Rule::new(IpRule::try_from("10.5.6.0/24").unwrap(), true),
+            ],
+            urls: vec![],
+        };
+        set.sort_rules();
+        // TODO: add ipv6
+        assert_eq!(set, RuleSet {
+            domains: vec![],
+            ips: vec![
+                Rule::new(IpRule::try_from("10.5.6.0/24").unwrap(), true),
+                Rule::new(IpRule::try_from("10.0.0.0/8").unwrap(), true),
+                Rule::new(IpRule::try_from("0.0.0.0/0").unwrap(), true),
+            ],
+            urls: vec![],
+        });
+    }
+
+    #[test]
+    fn test_rule_sort_urls() {
+        let mut set = RuleSet {
+            domains: vec![],
+            ips: vec![],
+            urls: vec![
+                Rule::new(UrlRule::try_from("http://example.com/foo/").unwrap(), true),
+                Rule::new(UrlRule::try_from("https://example.com/").unwrap(), true),
+                Rule::new(UrlRule::try_from("https://example.com/foo/bar/?asdf=1").unwrap(), true),
+            ],
+        };
+        set.sort_rules();
+        assert_eq!(set, RuleSet {
+            domains: vec![],
+            ips: vec![],
+            urls: vec![
+                Rule::new(UrlRule::try_from("https://example.com/foo/bar/?asdf=1").unwrap(), true),
+                Rule::new(UrlRule::try_from("http://example.com/foo/").unwrap(), true),
+                Rule::new(UrlRule::try_from("https://example.com/").unwrap(), true),
+            ],
+        });
+    }
+}

src/autonoscope/url.rs

@@ -0,0 +1,176 @@
+use crate::errors::*;
+use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::models::*;
+use std::convert::TryFrom;
+
+#[derive(Debug, PartialEq)]
+pub struct UrlRule {
+    url: String,
+    origin: url::Origin,
+    segments: Vec<String>,
+}
+
+impl ToString for UrlRule {
+    fn to_string(&self) -> String {
+        self.url.clone()
+    }
+}
+
+impl TryFrom<Autonoscope> for UrlRule {
+    type Error = Error;
+
+    fn try_from(x: Autonoscope) -> Result<UrlRule> {
+        UrlRule::try_from(x.value.as_str())
+    }
+}
+
+impl TryFrom<&str> for UrlRule {
+    type Error = Error;
+
+    fn try_from(x: &str) -> Result<UrlRule> {
+        let url = x.parse::<url::Url>()?;
+
+        let origin = url.origin();
+
+        let segments = url.path_segments()
+            .ok_or_else(|| format_err!("url can't have a base"))?
+            .filter(|x| !x.is_empty())
+            .map(String::from)
+            .collect();
+
+        Ok(UrlRule {
+            url: x.to_string(),
+            origin,
+            segments,
+        })
+    }
+}
+
+// TODO: there is no way to write a rule that matches all urls
+impl AutoRule<NewUrl> for UrlRule {
+    fn matches(&self, url: &NewUrl) -> Result<bool> {
+        self.matches(url.value.as_str())
+    }
+}
+
+impl AutoRule<str> for UrlRule {
+    fn matches(&self, url: &str) -> Result<bool> {
+        let url = url.parse::<url::Url>()?;
+
+        if url.origin() != self.origin {
+            return Ok(false);
+        }
+
+        let segments = url.path_segments()
+            .ok_or_else(|| format_err!("url can't have a base"))?
+            .filter(|x| !x.is_empty())
+            .collect::<Vec<_>>();
+
+        if self.segments.len() > segments.len() {
+            return Ok(false);
+        }
+
+        for (rule, path) in self.segments.iter().zip(segments.iter()) {
+            if rule != path {
+                return Ok(false);
+            }
+        }
+
+        Ok(true)
+    }
+}
+
+impl RulePrecision for UrlRule {
+    fn precision(&self) -> usize {
+        self.segments.len()
+    }
+}
+
+impl IntoRule for UrlRule {
+    fn into_rule(&self) -> (&'static str, String) {
+        ("url", self.to_string())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::convert::TryFrom;
+
+    #[test]
+    fn test_url_rule_explicit_origin() {
+        let rule = UrlRule::try_from("https://example.com").unwrap();
+        assert!(rule.matches("https://example.com:443/").unwrap());
+        assert_eq!(rule.precision(), 0);
+    }
+
+    #[test]
+    fn test_url_rule_scheme_mismatch() {
+        let rule = UrlRule::try_from("https://example.com").unwrap();
+        assert!(!rule.matches("http://example.com:443/").unwrap());
+        assert_eq!(rule.precision(), 0);
+    }
+
+    #[test]
+    fn test_url_rule_port_mismatch() {
+        let rule = UrlRule::try_from("https://example.com").unwrap();
+        assert!(!rule.matches("https://example.com:80/").unwrap());
+        assert_eq!(rule.precision(), 0);
+    }
+
+    #[test]
+    fn test_url_rule_subdomain_mismatch1() {
+        let rule = UrlRule::try_from("https://example.com").unwrap();
+        assert!(!rule.matches("https://www.example.com/").unwrap());
+        assert_eq!(rule.precision(), 0);
+    }
+
+    #[test]
+    fn test_url_rule_subdomain_mismatch2() {
+        let rule = UrlRule::try_from("https://www.example.com").unwrap();
+        assert!(!rule.matches("https://example.com/").unwrap());
+        assert_eq!(rule.precision(), 0);
+    }
+
+    #[test]
+    fn test_url_rule_ftp() {
+        let rule = UrlRule::try_from("ftp://www.example.com").unwrap();
+        assert!(!rule.matches("https://example.com/").unwrap());
+        assert_eq!(rule.precision(), 0);
+    }
+
+    #[test]
+    fn test_url_rule_outside_of_path() {
+        let rule = UrlRule::try_from("https://www.example.com/asset").unwrap();
+        assert!(!rule.matches("https://example.com/").unwrap());
+        assert_eq!(rule.precision(), 1);
+    }
+
+    #[test]
+    fn test_url_rule_path_match_implicit_slash() {
+        let rule = UrlRule::try_from("https://www.example.com/asset").unwrap();
+        assert!(rule.matches("https://www.example.com/asset/").unwrap());
+        assert_eq!(rule.precision(), 1);
+    }
+
+    #[test]
+    fn test_url_rule_path_match_explicit_slash() {
+        let rule = UrlRule::try_from("https://www.example.com/asset/").unwrap();
+        assert!(rule.matches("https://www.example.com/asset").unwrap());
+        assert_eq!(rule.precision(), 1);
+    }
+
+    #[test]
+    fn test_url_rule_in_folder_implicit_slash() {
+        let rule = UrlRule::try_from("https://www.example.com/asset").unwrap();
+        assert!(rule.matches("https://www.example.com/asset/style.css").unwrap());
+        assert_eq!(rule.precision(), 1);
+    }
+
+    #[test]
+    fn test_url_rule_in_folder_explicit_slash() {
+        let rule = UrlRule::try_from("https://www.example.com/asset/").unwrap();
+        assert!(rule.matches("https://www.example.com/asset/style.css").unwrap());
+        assert_eq!(rule.precision(), 1);
+    }
+}

src/blobs.rs

@@ -9,7 +9,7 @@ use bytes::Bytes;
 use serde::ser::{Serialize, Serializer};
 use serde::de::{self, Deserialize, Deserializer};
 use std::fs;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::mpsc;
 
@@ -45,12 +45,14 @@ impl Blob {
 impl EventWithCallback for Blob {
     type Payload = ();
 
+    #[inline(always)]
     fn with_callback(self, tx: mpsc::Sender<result::Result<Self::Payload, String>>) -> Event2 {
         Event2::Blob((self, tx))
     }
 }
 
 impl Serialize for Blob {
+    #[inline]
     fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
     where
         S: Serializer,
@@ -61,6 +63,7 @@ impl Serialize for Blob {
 }
 
 impl<'de> Deserialize<'de> for Blob {
+    #[inline]
     fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
     where
         D: Deserializer<'de>,
@@ -84,11 +87,13 @@ impl BlobStorage {
         }
     }
 
+    #[inline]
     pub fn workspace(workspace: &Workspace) -> Result<BlobStorage> {
         let path = paths::blobs_dir(workspace)?;
         Ok(BlobStorage::new(path))
     }
 
+    #[inline]
     pub fn join(&self, id: &str) -> Result<PathBuf> {
         if !id.chars().all(char::is_alphanumeric) {
             bail!("blob id contains invalid characters");
@@ -96,6 +101,11 @@ impl BlobStorage {
         Ok(self.path.join(id))
     }
 
+    #[inline(always)]
+    pub fn path(&self) -> &Path {
+        self.path.as_ref()
+    }
+
     pub fn load(&self, id: &str) -> Result<Blob> {
         let path = self.join(id)?;
 

src/cmd/add_cmd.rs

@@ -4,19 +4,19 @@ use crate::blobs::Blob;
 use crate::cmd::Cmd;
 use crate::gfx;
 use crate::models::*;
-use crate::shell::Readline;
+use crate::shell::Shell;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::utils;
 use crate::term;
 use std::fs;
+use std::net;
+use ipnetwork;
 use std::path::Path;
 use walkdir::WalkDir;
 
-
 #[derive(Debug, StructOpt)]
-#[structopt(author = "",
-            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
     #[structopt(subcommand)]
     subcommand: Target,
@@ -33,6 +33,12 @@ pub enum Target {
     /// Insert subdomain into the database
     #[structopt(name="subdomain")]
     Subdomain(AddSubdomain),
+    /// Insert ip network into the database
+    #[structopt(name="netblock")]
+    Netblock(AddNetblock),
+    /// Insert ip address into the database
+    #[structopt(name="ipaddr")]
+    IpAddr(AddIpAddr),
     /// Insert email into the database
     #[structopt(name="email")]
     Email(AddEmail),
@@ -57,10 +63,11 @@ pub enum Target {
 }
 
 impl Cmd for Args {
-    fn run(self, rl: &mut Readline) -> Result<()> {
+    fn run(self, rl: &mut Shell) -> Result<()> {
         match self.subcommand {
             Target::Domain(args) => args.insert(rl, self.dry_run),
             Target::Subdomain(args) => args.insert(rl, self.dry_run),
+            Target::IpAddr(args) => args.insert(rl, self.dry_run),
             Target::Email(args) => args.insert(rl, self.dry_run),
             Target::PhoneNumber(args) => args.insert(rl, self.dry_run),
             Target::Device(args) => args.insert(rl, self.dry_run),
@@ -68,19 +75,15 @@ impl Cmd for Args {
             Target::Account(args) => args.insert(rl, self.dry_run),
             Target::Breach(args) => args.insert(rl, self.dry_run),
             Target::Image(args) => args.insert(rl, self.dry_run),
+            Target::Netblock(args) => args.insert(rl, self.dry_run),
         }
     }
 }
 
-#[inline]
-pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
-    Args::run_str(rl, args)
-}
-
 trait IntoInsert: Sized {
-    fn into_insert(self, rl: &Readline) -> Result<Insert>;
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert>;
 
-    fn insert(self, rl: &Readline, dry_run: bool) -> Result<()> {
+    fn insert(self, rl: &mut Shell, dry_run: bool) -> Result<()> {
         let insert = self.into_insert(rl)?;
         if !dry_run {
             rl.db().insert_generic(insert)?;
@@ -95,22 +98,23 @@ pub struct AddDomain {
 }
 
 impl IntoInsert for AddDomain {
-    fn into_insert(self, rl: &Readline) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
         let domain = match self.domain {
             Some(domain) => domain,
             _ => utils::question("Domain")?,
         };
 
         // ensure input is a valid domain
-        let parsed_domain = rl.psl().parse_domain(&domain)
+        let dns_name = rl.psl()?.parse_dns_name(&domain)
             .map_err(|e| format_err!("Failed to parse domain: {}", e))?;
 
-        if Some(domain.as_str()) != parsed_domain.root() {
-            bail!("This is not a valid domain, might be a subdomain or tld");
+        if dns_name.fulldomain.is_some() {
+            bail!("Domain has an unexpected subdomain, add as a subdomain instead");
         }
 
         Ok(Insert::Domain(NewDomain {
             value: domain,
+            unscoped: false,
         }))
     }
 }
@@ -121,22 +125,19 @@ pub struct AddSubdomain {
 }
 
 impl IntoInsert for AddSubdomain {
-    fn into_insert(self, rl: &Readline) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
         let subdomain = match self.subdomain {
             Some(subdomain) => subdomain,
             _ => utils::question("Subdomain")?,
         };
 
-        let dns_name = rl.psl().parse_dns_name(&subdomain)
+        let dns_name = rl.psl()?.parse_dns_name(&subdomain)
             .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
 
-        let domain = dns_name.domain()
-            .ok_or_else(|| format_err!("Dns Name seems invalid"))?
-            .to_string();
-
         let domain_id = match rl.db().insert_struct(NewDomain {
-            value: domain,
-        })? {
+            value: dns_name.root,
+            unscoped: false,
+        }, true)? {
             Some((_, domain_id)) => domain_id,
             _ => bail!("Domain is out out of scope"),
         };
@@ -145,6 +146,46 @@ impl IntoInsert for AddSubdomain {
             domain_id,
             value: subdomain,
             resolvable: None,
+            unscoped: false,
+        }))
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AddIpAddr {
+    ipaddr: Option<net::IpAddr>,
+}
+
+impl IntoInsert for AddIpAddr {
+    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+        let ipaddr = match self.ipaddr {
+            Some(ipaddr) => ipaddr,
+            _ => {
+                let ipaddr = utils::question("IP address")?;
+                ipaddr.parse()?
+            },
+        };
+
+        let family = match ipaddr {
+            net::IpAddr::V4(_) => "4",
+            net::IpAddr::V6(_) => "6",
+        };
+
+        Ok(Insert::IpAddr(NewIpAddr {
+            family: family.to_string(),
+            value: ipaddr.to_string(),
+            continent: None,
+            continent_code: None,
+            country: None,
+            country_code: None,
+            city: None,
+            latitude: None,
+            longitude: None,
+            asn: None,
+            as_org: None,
+            description: None,
+            reverse_dns: None,
+            unscoped: false,
         }))
     }
 }
@@ -155,7 +196,7 @@ pub struct AddEmail {
 }
 
 impl IntoInsert for AddEmail {
-    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
         let email = match self.email {
             Some(email) => email,
             _ => utils::question("Email")?,
@@ -165,6 +206,7 @@ impl IntoInsert for AddEmail {
             value: email,
             displayname: None,
             valid: None,
+            unscoped: false,
         }))
     }
 }
@@ -176,7 +218,7 @@ pub struct AddPhoneNumber {
 }
 
 impl IntoInsert for AddPhoneNumber {
-    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
         let (phonenumber, name) = match self.phonenumber {
             Some(phonenumber) => (phonenumber, self.name),
             _ => {
@@ -198,6 +240,7 @@ impl IntoInsert for AddPhoneNumber {
             last_ported: None,
             caller_name: None,
             caller_type: None,
+            unscoped: false,
         }))
     }
 }
@@ -209,7 +252,7 @@ pub struct AddDevice {
 }
 
 impl IntoInsert for AddDevice {
-    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
         let (mac, name) = match self.mac {
             Some(mac) => {
                 (mac, self.name)
@@ -227,6 +270,7 @@ impl IntoInsert for AddDevice {
             hostname: None,
             vendor: None,
             last_seen: None,
+            unscoped: false,
         }))
     }
 }
@@ -239,7 +283,7 @@ pub struct AddNetwork {
 }
 
 impl IntoInsert for AddNetwork {
-    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
         let (network, latitude, longitude) = match self.network {
             Some(network) => (network, self.latitude, self.longitude),
             _ => {
@@ -254,6 +298,8 @@ impl IntoInsert for AddNetwork {
             value: network,
             latitude,
             longitude,
+            description: None,
+            unscoped: false,
         }))
     }
 }
@@ -265,7 +311,7 @@ pub struct AddAccount {
 }
 
 impl IntoInsert for AddAccount {
-    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
         let (service, username) = match (self.service, self.username) {
             (Some(service), Some(username)) => (service, username),
             _ => {
@@ -291,6 +337,10 @@ impl IntoInsert for AddAccount {
             email: None,
             url: None,
             last_seen: None,
+            birthday: None,
+            phonenumber: None,
+            profile_pic: None,
+            unscoped: false,
         }))
     }
 }
@@ -301,7 +351,7 @@ pub struct AddBreach {
 }
 
 impl IntoInsert for AddBreach {
-    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
         let name = match self.name {
             Some(name) => name,
             _ => {
@@ -312,6 +362,7 @@ impl IntoInsert for AddBreach {
 
         Ok(Insert::Breach(NewBreach {
             value: name,
+            unscoped: false,
         }))
     }
 }
@@ -322,11 +373,11 @@ pub struct AddImage {
 }
 
 impl IntoInsert for AddImage {
-    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
         unreachable!()
     }
 
-    fn insert(self, rl: &Readline, dry_run: bool) -> Result<()> {
+    fn insert(self, rl: &mut Shell, dry_run: bool) -> Result<()> {
         let paths = if self.paths.is_empty() {
             let path = utils::question("Path")?;
             vec![path]
@@ -404,6 +455,8 @@ impl IntoInsert for AddImage {
                         ahash: None,
                         dhash: None,
                         phash: None,
+
+                        unscoped: false,
                     }))?;
                 }
             }
@@ -412,3 +465,34 @@ impl IntoInsert for AddImage {
         Ok(())
     }
 }
+
+#[derive(Debug, StructOpt)]
+pub struct AddNetblock {
+    ipnet: Option<ipnetwork::IpNetwork>,
+}
+
+impl IntoInsert for AddNetblock {
+    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+        let ipnet = match self.ipnet {
+            Some(ipnet) => ipnet,
+            _ => {
+                let ipnet = utils::question("IP network")?;
+                ipnet.parse()?
+            },
+        };
+
+        let family = match ipnet {
+            ipnetwork::IpNetwork::V4(_) => "4",
+            ipnetwork::IpNetwork::V6(_) => "6",
+        };
+
+        Ok(Insert::Netblock(NewNetblock {
+            family: family.to_string(),
+            value: ipnet.to_string(),
+            asn: None,
+            as_org: None,
+            description: None,
+            unscoped: false,
+        }))
+    }
+}

src/cmd/autonoscope_cmd.rs

@@ -0,0 +1,71 @@
+use crate::errors::*;
+
+use crate::autonoscope;
+use crate::fmt::colors::*;
+use crate::shell::Shell;
+use std::fmt::Write;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Subcommand,
+}
+
+#[derive(Debug, StructOpt)]
+pub enum Subcommand {
+    #[structopt(name="add")]
+    Add(Add),
+    #[structopt(name="delete")]
+    Delete(Delete),
+    #[structopt(name="list")]
+    List,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Add {
+    object: autonoscope::RuleType,
+    value: String,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Delete {
+    object: autonoscope::RuleType,
+    value: String,
+}
+
+fn display_rule<T: Color>(object: &str, rule: &str) -> Result<()> {
+    let mut out = String::new();
+    T::display(&mut out, object)?;
+    write!(&mut out, " {:?}", rule)?;
+    println!("{}", out);
+    Ok(())
+}
+
+pub fn run_with_scope_param(rl: &mut Shell, args: Args, scoped: bool) -> Result<()> {
+    match args.subcommand {
+        Subcommand::Add(add) => {
+            rl.db_mut().autonoscope_add_rule(&add.object, &add.value, scoped)
+        },
+        Subcommand::Delete(delete) => {
+            rl.db_mut().autonoscope_delete_rule(&delete.object, &delete.value)
+        },
+        Subcommand::List => {
+            for (object, rule, scoped) in rl.db().autonoscope_rules() {
+                if scoped {
+                    display_rule::<Green>(&format!("  scope {}", object), &rule)?;
+                } else {
+                    display_rule::<Red>(&format!("noscope {}", object), &rule)?;
+                }
+            }
+            Ok(())
+        },
+    }
+}
+
+pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
+    let args = Args::from_iter_safe(args)?;
+    run_with_scope_param(rl, args, false)
+}

src/cmd/autoscope_cmd.rs

@@ -0,0 +1,13 @@
+use crate::errors::*;
+
+use crate::cmd::autonoscope_cmd;
+use crate::shell::Shell;
+use structopt::StructOpt;
+
+
+pub type Args = autonoscope_cmd::Args;
+
+pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
+    let args = Args::from_iter_safe(args)?;
+    autonoscope_cmd::run_with_scope_param(rl, args, true)
+}

src/cmd/delete_cmd.rs

@@ -2,7 +2,7 @@ use crate::errors::*;
 
 use crate::cmd::Cmd;
 use crate::filters::{Target, Filter};
-use crate::shell::Readline;
+use crate::shell::Shell;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::models::*;
@@ -10,15 +10,14 @@ use crate::term;
 
 
 #[derive(Debug, StructOpt)]
-#[structopt(author = "",
-            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
     #[structopt(subcommand)]
     subcommand: Target,
 }
 
 impl Cmd for Args {
-    fn run(self, rl: &mut Readline) -> Result<()> {
+    fn run(self, rl: &mut Shell) -> Result<()> {
         let rows = match &self.subcommand {
             Target::Domains(filter) => delete::<Domain>(rl, &filter),
             Target::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
@@ -31,6 +30,8 @@ impl Cmd for Args {
             Target::Accounts(filter) => delete::<Account>(rl, &filter),
             Target::Breaches(filter) => delete::<Breach>(rl, &filter),
             Target::Images(filter) => delete::<Image>(rl, &filter),
+            Target::Ports(filter) => delete::<Port>(rl, &filter),
+            Target::Netblocks(filter) => delete::<Netblock>(rl, &filter),
         }?;
         term::info(&format!("Deleted {} rows", rows));
         Ok(())
@@ -38,11 +39,11 @@ impl Cmd for Args {
 }
 
 #[inline]
-pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     Args::run_str(rl, args)
 }
 
 #[inline]
-fn delete<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<usize> {
+fn delete<T: Model + Detailed>(rl: &mut Shell, filter: &Filter) -> Result<usize> {
     T::delete(rl.db(), &filter.parse()?)
 }

src/cmd/export_cmd.rs

@@ -0,0 +1,128 @@
+use crate::errors::*;
+
+use crate::blobs::Blob;
+use crate::cmd::Cmd;
+use crate::db::ttl;
+use crate::shell::Shell;
+use serde_json;
+use serde::Serialize;
+use std::io::{self, Write};
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+use strum_macros::{EnumString, IntoStaticStr};
+use crate::models::*;
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Specify the export format
+    #[structopt(short="f", long="format", possible_values=Format::variants())]
+    format: Format,
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        ttl::reap_expired(rl.db())?;
+        match self.format {
+            Format::Json => export::<JsonFormat>(rl),
+            Format::JsonBlobs => export::<JsonBlobsFormat>(rl),
+        }
+    }
+}
+
+fn export<T: ExportFormat + Serialize>(rl: &mut Shell) -> Result<()> {
+    let export = T::load(rl)?;
+    let mut stdout = io::stdout();
+    serde_json::to_writer(&mut stdout, &export)?;
+    stdout.write_all(b"\n")?;
+    Ok(())
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[derive(EnumString, IntoStaticStr)]
+#[strum(serialize_all = "kebab_case")]
+pub enum Format {
+    Json,
+    JsonBlobs,
+}
+
+impl Format {
+    // TODO: this function should be generated by strum instead
+    #[inline]
+    fn variants() -> &'static [&'static str] {
+        &[
+            "json",
+            "json-blobs",
+        ]
+    }
+}
+
+trait ExportFormat {
+    fn load(rl: &mut Shell) -> Result<Box<Self>>;
+}
+
+#[derive(Serialize, Deserialize)]
+struct JsonFormat {
+    accounts: Vec<Account>,
+    breaches: Vec<Breach>,
+    breach_emails: Vec<BreachEmail>,
+    devices: Vec<Device>,
+    domains: Vec<Domain>,
+    emails: Vec<Email>,
+    images: Vec<Image>,
+    ipaddrs: Vec<IpAddr>,
+    netblocks: Vec<Netblock>,
+    networks: Vec<Netblock>,
+    network_devices: Vec<NetworkDevice>,
+    phonenumbers: Vec<PhoneNumber>,
+    ports: Vec<Port>,
+    subdomains: Vec<Subdomain>,
+    subdomain_ipaddrs: Vec<SubdomainIpAddr>,
+    urls: Vec<Url>,
+}
+
+impl ExportFormat for JsonFormat {
+    fn load(rl: &mut Shell) -> Result<Box<JsonFormat>> {
+        let db = rl.db();
+
+        Ok(Box::new(JsonFormat {
+            accounts: Account::list(db)?,
+            breaches: Breach::list(db)?,
+            breach_emails: BreachEmail::list(db)?,
+            devices: Device::list(db)?,
+            domains: Domain::list(db)?,
+            emails: Email::list(db)?,
+            images: Image::list(db)?,
+            ipaddrs: IpAddr::list(db)?,
+            netblocks: Netblock::list(db)?,
+            networks: Netblock::list(db)?,
+            network_devices: NetworkDevice::list(db)?,
+            phonenumbers: PhoneNumber::list(db)?,
+            ports: Port::list(db)?,
+            subdomains: Subdomain::list(db)?,
+            subdomain_ipaddrs: SubdomainIpAddr::list(db)?,
+            urls: Url::list(db)?,
+        }))
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+struct JsonBlobsFormat {
+    models: JsonFormat,
+    blobs: Vec<Blob>,
+}
+
+impl ExportFormat for JsonBlobsFormat {
+    fn load(rl: &mut Shell) -> Result<Box<JsonBlobsFormat>> {
+        let models = *JsonFormat::load(rl)?;
+        let storage = rl.blobs();
+        let blobs = storage.list()?
+            .into_iter()
+            .map(|id| storage.load(&id))
+            .collect::<Result<Vec<_>>>()?;
+        Ok(Box::new(JsonBlobsFormat {
+            models,
+            blobs,
+        }))
+    }
+}

src/cmd/fsck_cmd.rs

@@ -2,7 +2,7 @@ use crate::errors::*;
 
 use crate::blobs::Blob;
 use crate::cmd::Cmd;
-use crate::shell::Readline;
+use crate::shell::Shell;
 use crate::term;
 use crate::worker;
 use crate::models::*;
@@ -24,7 +24,7 @@ pub struct Args {
 }
 
 impl Cmd for Args {
-    fn run(self, rl: &mut Readline) -> Result<()> {
+    fn run(self, rl: &mut Shell) -> Result<()> {
         let blobs = rl.blobs();
 
         let hashset = worker::spawn_fn("Building reference set...", || {

src/cmd/help_cmd.rs

@@ -1,6 +1,6 @@
 use crate::errors::*;
 
-use crate::shell::Readline;
+use crate::shell::Shell;
 
 
 #[inline]
@@ -8,22 +8,24 @@ fn help(name: &str, descr: &str) {
     println!("    \x1b[32m{:13}\x1b[0m {}", name, descr);
 }
 
-pub fn run(_rl: &mut Readline, _args: &[String]) -> Result<()> {
+pub fn run(_rl: &mut Shell, _args: &[String]) -> Result<()> {
 
     println!("\n\x1b[33mCOMMANDS:\x1b[0m");
-    help("add",        "Add new entities to the database");
-    help("delete",     "Delete entities from the database");
-    help("keyring",    "Manage saved credentials");
-    help("mod",        "Manage installed modules");
-    help("noscope",    "Exclude entities from scope");
-    help("quickstart", "Install all featured modules");
-    help("run",        "Run the currently selected module");
-    help("scope",      "Include entities in the scope again");
-    help("select",     "Select entities from the database");
-    help("target",     "Preview targeted entities or narrow them down");
-    help("use",        "Select a module");
-    help("workspace",  "Switch to a different workspace");
-    help("help",       "Prints this message");
+    help("add",         "Add new entities to the database");
+    help("autonoscope", "Manage rules to automatically remove entities from scope");
+    help("autoscope",   "Manage rules to automatically add entities to scope");
+    help("delete",      "Delete entities from the database");
+    help("keyring",     "Manage saved credentials");
+    help("mod",         "Manage installed modules");
+    help("noscope",     "Exclude entities from scope");
+    help("quickstart",  "Install all featured modules");
+    help("run",         "Run the currently selected module");
+    help("scope",       "Include entities in the scope again");
+    help("select",      "Select entities from the database");
+    help("target",      "Preview targeted entities or narrow them down");
+    help("use",         "Select a module");
+    help("workspace",   "Switch to a different workspace");
+    help("help",        "Prints this message");
     println!("\nRun <command> -h for more help.\n");
 
     Ok(())

src/cmd/keyring_cmd.rs

@@ -1,15 +1,14 @@
 use crate::errors::*;
 
 use crate::keyring::{KeyName, KeyRing};
-use crate::shell::Readline;
+use crate::shell::Shell;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::utils;
 
 
 #[derive(Debug, StructOpt)]
-#[structopt(author = "",
-            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub enum Args {
     #[structopt(name="add")]
     /// Add a new key to the keyring
@@ -50,28 +49,35 @@ pub struct KeyRingList {
     namespace: Option<String>,
 }
 
-pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
     match args {
-        Args::Add(add) => keyring_add(rl.keyring_mut(), add),
-        Args::Delete(delete) => keyring_delete(rl.keyring_mut(), delete),
+        Args::Add(add) => keyring_add(rl, add),
+        Args::Delete(delete) => keyring_delete(rl, delete),
         Args::Get(get) => keyring_get(rl.keyring(), &get),
         Args::List(list) => keyring_list(rl.keyring(), list),
     }
 }
 
-fn keyring_add(keyring: &mut KeyRing, add: KeyRingAdd) -> Result<()> {
+fn keyring_add(rl: &mut Shell, add: KeyRingAdd) -> Result<()> {
+    let keyring = rl.keyring_mut();
+
     // TODO: there's no non-interactive way to add a key without a secret key
     let secret = match add.secret {
         Some(secret) => Some(secret),
         None => utils::question_opt("Secretkey")?,
     };
 
-    keyring.insert(add.key, secret)
+    keyring.insert(add.key, secret)?;
+    rl.reload_keyring_cache();
+    Ok(())
 }
 
-fn keyring_delete(keyring: &mut KeyRing, delete: KeyRingDelete) -> Result<()> {
-    keyring.delete(delete.key)
+fn keyring_delete(rl: &mut Shell, delete: KeyRingDelete) -> Result<()> {
+    let keyring = rl.keyring_mut();
+    keyring.delete(delete.key)?;
+    rl.reload_keyring_cache();
+    Ok(())
 }
 
 fn keyring_get(keyring: &KeyRing, get: &KeyRingGet) -> Result<()> {