Release v0.11.0

Add image forensics support

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.9.0"
+version = "0.11.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -9,15 +9,24 @@ categories = ["command-line-utilities"]
 readme = "README.md"
 edition = "2018"
 
+[profile.release]
+# skip lto to avoid compiler bug:
+# https://github.com/kpcyrd/sn0int/issues/77
+# https://github.com/rust-lang/rust/issues/58674
+opt-level = 1
+lto = false
+
 [badges]
 travis-ci = { repository = "kpcyrd/sn0int" }
 
 [workspace]
-members = ["sn0int-registry/sn0int-common",
-           "sn0int-registry"]
+# rocket is broken and blocks the whole workspace, removing
+#members = ["sn0int-registry/sn0int-common",
+#           "sn0int-registry"]
+members = ["sn0int-registry/sn0int-common"]
 
 [dependencies]
-sn0int-common = { version="0.4.0", path="sn0int-registry/sn0int-common" }
+sn0int-common = { version="0.6.0", path="sn0int-registry/sn0int-common" }
 rustyline = "3"
 log = "0.4"
 env_logger = "0.6"
@@ -35,7 +44,7 @@ chrono = { version = "0.4", features = ["serde"] }
 dirs = "1.0"
 url = "1.7"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.8"
+chrootable-https = "0.9"
 base64 = "0.10"
 kuchiki = "0.7.2"
 serde_urlencoded = "0.5"
@@ -49,7 +58,7 @@ separator = "0.4"
 maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.2"
 regex = "1.0"
-toml = "0.4"
+toml = "0.5"
 maxminddb = "0.13"
 tar = "0.4.17"
 libflate = "0.1.14"
@@ -60,10 +69,26 @@ nom = "4.1.1"
 atty = "0.2"
 bufstream = "0.1.4"
 tokio = "0.1.14"
+semver = "0.9"
+bytes = "0.4"
+
+digest = "0.8.0"
+hex = "0.3.1"
+bs58 = "0.2.2"
+blake2 = "0.8.0"
+md-5 = "0.8.0"
+sha-1 = "0.8.1"
+sha2 = "0.8.0"
+
+image = "0.21"
+kamadak-exif = "0.3.1"
+walkdir = "2.2"
+nude = "0.1.0"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
-syscallz = "0.8"
+#syscallz = { path="../syscallz-rs" }
+syscallz = "0.11"
 nix = "0.13"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
@@ -71,4 +96,10 @@ pledge = "0.3.1"
 unveil = "0.2.0"
 
 [dev-dependencies]
-boxxy = "0.8"
+#boxxy = { path = "../boxxy-rs" }
+boxxy = "0.9"
+tempfile = "3.0"
+
+#[patch.crates-io]
+#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }
+#rocket_contrib = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }

Dockerfile

@@ -1,13 +1,14 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM debian
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

ISSUE_TEMPLATE.md

@@ -0,0 +1,29 @@
+<!--
+Hello!
+
+If you want to report a bug we added some common questions below that help us analyse your issue faster.
+
+All of these are optional so feel free to remove anything that doesn't apply.
+-->
+
+please describe your issue here
+
+---
+
+## Versions
+
+- **rustc --version:**
+- **cargo --version:**
+- **sn0int --version:**
+- **uname -a:**
+
+## Environment
+
+- **Operating System/Distro:**
+- **Installed from (source/apt/pacman/brew/docker):**
+
+<!--
+Thank you!
+
+We'll try to respond as quickly as possible.
+-->

Makefile

@@ -19,3 +19,10 @@ update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt
 	get-iab -v -u http://standards-oui.ieee.org/iab/iab.txt -f data/ieee-iab.txt
 	rm -f data/ieee-*.txt.bak
+
+docs:
+	$(MAKE) -C docs html
+	contrib/html-toc2md.pl README.md docs/_build/html/index.html > README2.md
+	mv README2.md README.md
+
+.PHONY: check force-check test update docs

README.md

@@ -1,4 +1,4 @@
-# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs]
+# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
 [travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
 [travis]:       https://travis-ci.org/kpcyrd/sn0int
@@ -6,6 +6,12 @@
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
 [docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
+[irc-img]:      https://img.shields.io/badge/hackint-%23sn0int-blue.svg
+[irc]:          https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int
+[mastodon-img]: https://img.shields.io/badge/mastodon-chaos.social-blue.svg
+[mastodon]:     https://chaos.social/@sn0int
+[registry-img]: https://img.shields.io/website/https/sn0int.com.svg?label=registry
+[registry]:     https://sn0int.com/
 
 sn0int is a semi-automatic OSINT framework and package manager. It was built
 for IT security professionals and bug hunters to gather intelligence about a
@@ -15,36 +21,52 @@ unified format for followup investigations.
 
 Among other things, sn0int is currently able to:
 
-- Harvest subdomains from certificate transparency logs
-- Harvest subdomains from various passive dns logs
-- Sift through subdomain results for publicly accessible websites
-- Harvest emails from pgp keyservers
-- Enrich ip addresses with ASN and geoip info
-- Harvest subdomains from the wayback machine
+- Harvest subdomains from certificate transparency logs and passive dns
+- Enrich ip addresses with asn and geoip info
+- Harvest emails from pgp keyservers and whois
+- Discover compromised logins in breaches
+- Find somebody's profiles across the internet
+- Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
-- Bruteforce interesting urls
+- Harvest data and images from instagram profiles
+- Scan images for nudity
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
-and is fully opensource.  None of the investigations listed above are hardcoded
+and is fully opensource. None of the investigations listed above are hardcoded
 in the source, instead those are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
 you to ship updates for your modules on your own since you don't need to send a
 pull request.
 
-Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
+For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
 [![asciicast](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB.svg)](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB)
 
+## Installation
+
+Archlinux
+
+    pacman -S sn0int
+
+Mac OSX
+
+    brew install sn0int
+
+For everything else please have a look at the [detailed list][1].
+
+[1]: https://sn0int.readthedocs.io/en/latest/install.html
+
 ## Getting started
 
 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
-  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
-  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
-  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
-  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
+  - [Debian testing/Debian sid/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-testing-debian-sid-kali)
+  - [Ubuntu/Debian stable](https://sn0int.readthedocs.io/en/latest/install.html#ubuntu-debian-stable)
+  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
+  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
 - [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
   - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
@@ -55,23 +77,44 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
 - [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
   - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
+- [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/structs.html#domains)
+  - [Subdomains](https://sn0int.readthedocs.io/en/latest/structs.html#subdomains)
+  - [IpAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#ipaddrs)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/structs.html#urls)
+  - [Emails](https://sn0int.readthedocs.io/en/latest/structs.html#emails)
+  - [Phonenumbers](https://sn0int.readthedocs.io/en/latest/structs.html#phonenumbers)
+  - [Devices](https://sn0int.readthedocs.io/en/latest/structs.html#devices)
+  - [Networks](https://sn0int.readthedocs.io/en/latest/structs.html#networks)
+  - [Accounts](https://sn0int.readthedocs.io/en/latest/structs.html#accounts)
+  - [Breaches](https://sn0int.readthedocs.io/en/latest/structs.html#breaches)
+  - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
+  - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
+- [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
+- [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
+- [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
   - [Using access keys as source argument](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-as-source-argument)
 - [Configuration](https://sn0int.readthedocs.io/en/latest/config.html)
-  - [Configuring a proxy](https://sn0int.readthedocs.io/en/latest/config.html#configuring-a-proxy)
+  - [\[core\]](https://sn0int.readthedocs.io/en/latest/config.html#core)
+  - [\[namespaces\]](https://sn0int.readthedocs.io/en/latest/config.html#namespaces)
+  - [\[network\]](https://sn0int.readthedocs.io/en/latest/config.html#network)
 - [Sandbox](https://sn0int.readthedocs.io/en/latest/sandbox.html)
   - [Linux](https://sn0int.readthedocs.io/en/latest/sandbox.html#linux)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/sandbox.html#openbsd)
   - [IPC Protocol](https://sn0int.readthedocs.io/en/latest/sandbox.html#ipc-protocol)
   - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
+  - [Diagnosing a sandbox failure](https://sn0int.readthedocs.io/en/latest/sandbox.html#diagnosing-a-sandbox-failure)
 - [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
   - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
+  - [create_blob](https://sn0int.readthedocs.io/en/latest/reference.html#create-blob)
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
@@ -86,18 +129,25 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
   - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
   - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
+  - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
+  - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [sha1](https://sn0int.readthedocs.io/en/latest/reference.html#sha1)
+  - [sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-256)
+  - [sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-512)
   - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
@@ -122,6 +172,23 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
 
+## Rationale
+
+This tool was written for companies to help them understand their attack
+surface from a blackbox point of view. It's often difficult to understand that
+something is easier to discover than some people assume, putting them at risk
+of false security.
+
+It's also designed to be useful for red team assessments and bug bounties,
+which also help companies to identify weaknesses that could result in a
+compromise.
+
+Some functionality was written to do the same thing for individuals to raise
+awareness about personal attack surface, privacy and how much data is publicly
+available. These issues are often out of scope in bug bounties and sometimes by
+design. We believe that blaming the user is the wrong approach and these issues
+should be addressed at the root cause by the people designing those systems.
+
 ## License
 
 GPLv3+

ci/boxxy_stage1.txt

@@ -1,3 +1,5 @@
 stage1
 ls
+echo checkpoint1
+id
 echo sandbox fail

ci/integration.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+import subprocess
+from subprocess import DEVNULL, PIPE
+import tempfile
+import json
+import sys
+
+
+def _sn0int(tempdir, binary, args, piped_stdout=False):
+    return subprocess.Popen(
+        ['/usr/bin/env', 'HOME='+tempdir, binary] + args,
+        stdin=PIPE,
+        stdout=PIPE if piped_stdout else None,
+    )
+
+
+def sn0int(tempdir, binary, cmds):
+    p = _sn0int(tempdir, binary, [])
+    for cmd in cmds:
+        p.stdin.write((cmd + '\n').encode('utf-8'))
+    p.communicate()
+    if p.returncode != 0:
+        raise Exception('process failed')
+
+
+def sn0int_select(tempdir, binary, query):
+    p = _sn0int(tempdir, binary, ['select', '--json'] + query, piped_stdout=True)
+    stdout, _ = p.communicate()
+    lines = filter(None, stdout.decode('utf-8').split('\n'))
+    return [json.loads(x) for x in lines]
+
+
+def main(tempdir, binary):
+    print('[*] setting up workspace')
+    sn0int(tempdir, binary, [])
+
+    print('[*] adding domain')
+    sn0int(tempdir, binary, [
+        'add domain',
+        'example.com',
+        'select domains',
+    ])
+
+    print('[*] testing db for domain')
+    domains = sn0int_select(tempdir, binary, ['domains'])
+    assert domains == [{'id': 1, 'value': 'example.com', 'unscoped': False}]
+
+    print('[*] installing modules')
+    sn0int(tempdir, binary, [
+        'mod install kpcyrd/ctlogs',
+        'mod install kpcyrd/dns-resolve',
+        'mod install kpcyrd/url-scan',
+        'mod install kpcyrd/geoip',
+    ])
+
+    print('[*] running ctlogs')
+    sn0int(tempdir, binary, [
+        'use ctlogs',
+        'run',
+        'select subdomains',
+    ])
+
+    print('[*] testing db for subdomains')
+    subdomains = sn0int_select(tempdir, binary, ['subdomains'])
+    assert {x['value'] for x in subdomains} == {
+        'www.example.com',
+        'm.example.com',
+        'dev.example.com',
+        'products.example.com',
+        'support.example.com',
+    }
+
+    print('[*] running dns-resolve')
+    sn0int(tempdir, binary, [
+        'use dns-resolve',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs')
+    ipaddrs = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert len(ipaddrs) >= 1
+
+    print('[*] running url-scan')
+    sn0int(tempdir, binary, [
+        'use url-scan',
+        'run',
+        'select urls',
+    ])
+
+    print('[*] testing db for urls')
+    urls = sn0int_select(tempdir, binary, ['urls'])
+    assert {(x['value'], x['status']) for x in urls} == {
+        ('http://www.example.com/', 200),
+        ('https://www.example.com/', 200),
+    }
+
+    print('[*] running geoip')
+    sn0int(tempdir, binary, [
+        'use geoip',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs again')
+    ipaddrs2 = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert ipaddrs != ipaddrs2
+
+    print('')
+    print('\t###########')
+    print('\t# SUCCESS #')
+    print('\t###########')
+    print('')
+
+
+if __name__ == '__main__':
+    try:
+        binary = sys.argv[1]
+    except IndexError:
+        print('Usage: %s target/release/sn0int' % sys.argv[0])
+    else:
+        with tempfile.TemporaryDirectory(prefix='sn0int-') as tempdir:
+            main(tempdir, binary)

contrib/html-toc2md.pl

@@ -0,0 +1,39 @@
+#!/usr/bin/env perl
+use strict; use warnings;
+
+my ($readme, $toc) = @ARGV;
+defined $readme or die 'missing readme path';
+defined $toc or die 'missing toc path';
+
+open(my $r, "<$readme") or die 'failed to open readme';
+open(my $t, "<$toc") or die 'failed to open toc';
+
+my $re = qr/^\s*- \[.+\]\(https:\/\/sn0int.readthedocs.io\/en\/.+\)$/;
+
+# pass through start of readme
+while (<$r>) {
+    last if ($_ =~ $re);
+    print $_;
+}
+
+# skip toc
+while (<$r>) {
+    last unless ($_ =~ $re);
+}
+
+# generate new toc
+while (my $line = <$t>) {
+    if ($line =~ /toctree-l(\d).*href="([^"]+)">(.+)<\/a/) {
+        my $space = $1;
+        my $section = $2;
+        my $label = $3;
+        $label =~ s/([\[\]])/\\$1/g;
+        print $space==2?"  ":"", "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
+    }
+}
+print;
+
+# pass through end of readme
+while (<$r>) {
+    print $_;
+}

contrib/html-toc2md.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-perl -n -e '/toctree-l(\d).*href="([^"]+)">(.+)<\/a/ && print $1==2?"  ":"", "- [$3](https://sn0int.readthedocs.io/en/latest/$2)\n"' < docs/_build/html/index.html

docs/config.rst

@@ -1,17 +1,54 @@
 Configuration
 =============
 
-This file documents the config file at ``~/.config/sn0int.toml``. By default
-this file does not exist and a default configuration is used instead.
+This section documents the config file. By default this file does not exist and
+a default configuration is used instead.
 
-Configuring a proxy
--------------------
+Linux/BSD
+    ``~/.config/sn0int.toml``
+
+OSX
+    ``~/Library/Preferences/sn0int.toml``
+
+Windows
+    ``%APPDATA%/sn0int.toml``
+
+[core]
+------
+
+``registry``
+    Configure the registry you want to use. Defaults to ``https://sn0int.com``.
+``no-autoupdate``
+    sn0int is going to check if your modules are outdated during startout once
+    a week. Set this option to ``true`` to disable this.
+
+[namespaces]
+------------------
+
+By default sn0int modules are assumed to be installed from the registry. You
+may want to keep a local directory with private modules, especially during
+development. You can configure a folder that contains modules that aren't
+managed by sn0int by adding a namespace section to the config file::
+
+    [namespaces]
+    foo = "/opt/sn0int/foo"
+    bar = "~/repos/a/b/c/sn0int-modules"
+
+This is going to load modules from these two folders and register them in the
+``foo`` and ``bar`` namespace.
+
+Note that sn0int is also going to assume that symlinks in
+``~/.local/share/sn0int/modules`` and folders containing a ``.git`` folder are
+externally managed.
+
+[network]
+---------
 
 To enable a proxy, add the following to your config file::
 
     [network]
     proxy = "127.0.0.1:9050"
 
-This forces everything through tor and restricts all other functions that
-depend on the network. For example the ``dns`` function is fully disabled if a
-proxy is configured.
+This forces everything through tor (or any other socks5 proxy) and restricts
+all other functions that depend on the network. For example the ``dns``
+function is fully disabled if a proxy is configured.

docs/index.rst

@@ -39,6 +39,7 @@ Getting Started
    usage
    scripting
    database
+   structs
    keyring
    config
    sandbox

docs/install.rst

@@ -10,22 +10,38 @@ Archlinux
 
     $ pacman -S sn0int
 
-Debian/Ubuntu/Kali
-------------------
+Mac OSX
+-------
 
 .. code-block:: bash
 
-    $ apt install libsqlite3-dev libseccomp-dev
+    $ brew install sn0int
+
+Debian testing/Debian sid/Kali
+------------------------------
+
+Note that debian `doesn't ship the geoip2-database
+<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
+download them automatically during the first run.
+
+.. code-block:: bash
+
+    $ apt install build-essential cargo libsqlite3-dev libseccomp-dev publicsuffix
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .
 
-Alpine
-------
+Ubuntu/Debian stable
+--------------------
+
+cargo in the repos is too old and the build is `going to fail
+<https://github.com/kpcyrd/sn0int/issues/68>`_. You should either install the
+most recent rust version with `rustup <https://rustup.rs/>`_ or use the docker
+instructions instead.
 
 .. code-block:: bash
 
-    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
+    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .
@@ -37,6 +53,16 @@ Docker
 
     $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
 
+Alpine
+------
+
+.. code-block:: bash
+
+    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f --path .
+
 OpenBSD
 -------
 
@@ -47,15 +73,6 @@ OpenBSD
     $ cd sn0int
     $ cargo install -f --path .
 
-Mac OSX
--------
-
-.. code-block:: bash
-
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
-
 Windows
 -------
 

docs/keyring.rst

@@ -52,8 +52,8 @@ If the user granted us access to those keys we can read them with ``keyring``:
 .. code-block:: lua
 
     creds = keyring('aws')
-    print(creds[1]['accesskey'])
-    print(creds[1]['secretkey'])
+    debug(creds[1]['access_key'])
+    debug(creds[1]['secret_key'])
 
 This returns a list of all keys in that namespace. Any empty list is returned
 if the user doesn't have any keys in that namespace.

docs/reference.rst

@@ -13,6 +13,18 @@ Clear the last recorded error from the internal state. See also last_err_.
         clear_err()
     end
 
+create_blob
+-----------
+
+Push a byte array into persistent blob storage. This allows passing those bytes
+to functions operating on blob storage. Returns a blob identifier that is
+deterministic based on the blob content. Blobs are immutable.
+
+.. code-block:: lua
+
+    blob = create_blob("some bytes")
+    debug(blob)
+
 datetime
 --------
 
@@ -210,6 +222,9 @@ options are set. The following options are available:
   The request body that should be form encoded.
 ``body``
   The raw request body as string.
+``into_blob``
+  If true, the response body is stored in blob storage and a blob reference is
+  returned as ``blob`` instead of the full body.
 
 This function may fail.
 
@@ -237,6 +252,9 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``blob``
+  If ``into_blob`` was enabled for the request the body is downloaded into blob
+  storage with a reference to the body in this field.
 
 .. code-block:: lua
 
@@ -250,6 +268,41 @@ the following keys:
     if last_err() then return end
     if resp["status"] ~= 200 then return "invalid status code" end
 
+img_load
+--------
+
+Attempt to decode a blob as an image and return some basic metadata like the
+mime type, height and width.
+
+.. code-block:: lua
+
+    img = img_load(blob)
+    if last_err() then return end
+    debug(img)
+
+img_exif
+--------
+
+Extract exif metadata from an image.
+
+.. code-block:: lua
+
+    exif = img_exif(blob)
+    if last_err() then return end
+    debug(exif)
+
+img_nudity
+----------
+
+Classify an image for nudity. The score goes from 0 to 2. A score above 1 means
+nudity has been detected.
+
+.. code-block:: lua
+
+    nudity = img_nudity(blob)
+    if last_err() then return end
+    debug(nudity)
+
 info
 ----
 
@@ -318,6 +371,15 @@ otherwise.
         return
     end
 
+md5
+---
+
+Hash a byte array with md5 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(md5("\x00\xff"))
+
 pgp_pubkey
 ----------
 
@@ -390,12 +452,12 @@ psl_domain_from_dns_name
 ------------------------
 
 Returns the parent domain according to the public suffix list. For
-``www.a.b.c.d.example.com`` this is going to be ``example.com``.
+``www.a.b.c.d.example.co.uk`` this is going to be ``example.co.uk``.
 
 .. code-block:: lua
 
-    domain = psl_domain_from_dns_name('www.a.b.c.d.example.com')
-    print(domain == 'example.com')
+    domain = psl_domain_from_dns_name('www.a.b.c.d.example.co.uk')
+    print(domain == 'example.co.uk')
 
 regex_find
 ----------
@@ -430,6 +492,33 @@ Same as regex_find_, but returns all matches.
     print(m[3][1] == 'ef')
     print(m[3][2] == 'f')
 
+sha1
+----
+
+Hash a byte array with sha1 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha1("\x00\xff"))
+
+sha2_256
+--------
+
+Hash a byte array with sha2_256 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha2_256("\x00\xff"))
+
+sha2_512
+--------
+
+Hash a byte array with sha2_512 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha2_512("\x00\xff"))
+
 sleep
 -----
 

docs/sandbox.rst

@@ -111,3 +111,37 @@ There are some limitations that you should be aware:
   means scripts have access to your local network, the internet and also your
   localhost loopback interface.
 - If chroot is unavailable an attacker could connect to unix domain sockets.
+
+Diagnosing a sandbox failure
+----------------------------
+
+You might experience a sandbox failure, especially on architectures that are
+less popular. This usually looks like this::
+
+    [sn0int][example][kpcyrd/ctlogs] > run
+    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [+] Finished kpcyrd/ctlogs (1 errors)
+
+A module that never finishes could also mean an IO thread inside the worker got
+killed by the sandbox.
+
+You can try to diagnose this yourself with strace::
+
+    strace -f sn0int run -vv ctlogs 2>&1 | tee strace.log
+
+Open ``strace.log``, look out for syscalls that didn't return by searching for
+``= ?`` and ignore calls to exit and similar. You are looking for something
+like this::
+
+    seccomp(SECCOMP_SET_MODE_FILTER, 0, {len=48, filter=0xdd59094e490}) = 0
+    write(1, "[+] activated!\n", 15[+] activated!
+    )        = 15
+    getresuid( <unfinished ...>)            = ?
+    +++ killed by SIGSYS (core dumped) +++
+
+This would indicate a call to ``getresuid`` which was not allowed by the
+seccomp filter.
+
+If you don't want to diagnose this yourself open a new bug report with as much
+information as possible, specifically which distro, which release and which
+architecture you're using.

docs/scripting.rst

@@ -13,13 +13,29 @@ free to change that to something else::
 
     $ git init ~/repos/sn0int-modules
     $ cd ~/repos/sn0int-modules
-    $ ln -s "$PWD" ~/.local/share/sn0int/modules/$YOUR_GITHUB_NAME
+
+We need to add this folder to the sn0int config file so it's correctly detected
+when starting sn0int. Open the `config file <config.html>`_ in your prefered
+editor. Note that the file does not exist by default and the path is different
+depending on your operating system. On linux you would open the config file
+with::
+
+    $ vim ~/.config/sn0int.toml
+
+Add the follwing::
+
+    [namespaces]
+    your_github_name = "~/repos/sn0int-modules"
 
 Every module we're adding to ``~/repos/sn0int-modules`` is now going to be
 picked up by sn0int.
 
-Let's add our first module by opening ``~/repos/sn0int-modules/first.lua``.
-There's a bit of boilerplate that every module needs to load successfully:
+Make sure you're still in the right folder and add your first module::
+
+    sn0int new first.lua
+
+This is going to generate some boilerplate for you that every module needs to
+load successfully. Afterwards we can edit it like this:
 
 .. code-block:: lua
 
@@ -188,3 +204,40 @@ your identity.
 Afterwards publish your module with::
 
     sn0int publish ./first.lua
+
+Reading data from stdin
+-----------------------
+
+Sometimes you need to read data that can't be easily accessed from within the
+sandbox, like output of other programms or file content. In that case you can
+write a module that reads from stdin:
+
+.. code-block:: lua
+
+    -- Description: Read from stdin
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+
+    function run()
+        while true do
+            x = stdin_readline()
+            if x == nil then
+                break
+            end
+            info(x)
+        end
+    end
+
+Write it to a file and run it like this::
+
+    % echo hello | sn0int run --stdin -vvf stdin.lua
+    [*] anonymous/stdin                                   : "hello\n"
+    [+] Finished anonymous/stdin
+    %
+
+This is going to read one line at a time and allows you to process it with
+regular expressions and add data to the database.
+
+.. note::
+   If you get an error like ``Failed to read stdin: "stdin is unavailable"``
+   make sure the ``--stdin`` flag is set.

docs/structs.rst

@@ -0,0 +1,254 @@
+Structs
+=======
+
+This section describes all supported structs in depth. Please refer to this
+section if in doubt about the correct usage of fields to ensure
+interoperability between modules.
+
+Domains
+-------
+
+Represents a registerable domain as defined by the `public suffix list
+<https://publicsuffix.org/>`_. If in doubt check `psl_domain_from_dns_name
+<reference.html#psl-domain-from-dns-name>`_.
+
+``value``
+  The domain name, like ``example.co.uk``.
+
+Subdomains
+----------
+
+A subdomain of a `domain <#domains>`_. The depth is arbitrary, so
+``foo.example.co.uk`` and ``foo.bar.example.co.uk`` are both valid subdomains
+of ``example.co.uk``.
+
+``value``
+  The subdomain, like ``foo.bar.example.co.uk``.
+``domain_id``
+  The numeric id of a domain struct.
+``resolvable``
+  Whether the subdomain can be resolved to a A/AAAA record. nil if unknown.
+
+IpAddrs
+-------
+
+An ip address. Note that most of these fields are geoip related and an
+approximation instead of an actual location.
+
+``value``
+    The ip address.
+``family``
+    The address family of the ip address, either ``4`` or ``6``.
+``continent``
+    The continent associated with this ip address.
+``continent_code``
+    The continent code of the ``continent`` field, eg ``NA``.
+``country``
+    The country associated with this ip address.
+``country_code``
+    The country code of the ``country`` field, eg ``US``.
+``city``
+    The city associated with this ip address.
+``latitude``
+    Latitude associated with this ip address.
+``longitude``
+    Longitude associated with this ip address.
+``asn``
+    The number of the autonomous system this ip belongs to.
+``as_org``
+    The organization of the autonomous system this ip belongs to.
+``description``
+    This field is sn0int internal if we have additional information about this
+    ip address, for example technical identifiers from aws.
+``reverse_dns``
+    The reverse dns name setup for this ip address.
+
+URLs
+----
+
+``subdomain_id``
+    The numeric id of a subdomain struct.
+``value``
+    The url, including a schema, hostname and path.
+``status``
+    The http status code, like ``200``.
+``body``
+    The raw response body. This can be any mime type.
+``online``
+    Whether or not the url gives a http response (even if it's an error).
+``title``
+    The parsed ``<title>`` of the page, if available.
+``redirect``
+    If the server replied with a redirect, this is the url it redirected to.
+
+Emails
+------
+
+``value``
+    The email address.
+``displayname``
+    The display name of a given email address: ``this is the name <foo@example.com>``.
+``valid``
+    Whether that email address is valid or has been disabled.
+
+Phonenumbers
+------------
+
+``value``
+    The phone number in E.164 format (+491234567)
+``name``
+    An alias we can assign to this phone number. This alias is sn0int internal.
+``valid``
+    Whether the number is assigned to a customer.
+``last_online``
+    The last time this number has been online.
+``country``
+    The country this number is associated with.
+``carrier``
+    The name of the carrier this numer is registered with.
+``line``
+    The type of the phone number, can be ``landline``, ``mobile`` or ``voip``.
+``is_ported``
+    Whether this number has been ported to a different carrier.
+``last_ported``
+    The last time this number has been ported.
+``caller_name``
+    The name of the owner of the phone number.
+``caller_type``
+    The type of caller, eg ``business`` or ``consumer``.
+
+Devices
+-------
+
+``value``
+    The devices mac address or another identifier if needed.
+``name``
+    An alias we can assign to this device. This alias is sn0int internal.
+``hostname``
+    The hostname configured on the device.
+``vendor``
+    The hardware vendor of the device. This is usually derived from the mac
+    address.
+``last_seen``
+    The last time we've observed the device somewhere.
+
+Networks
+--------
+
+A wired or wireless network at a specific location that a device could be
+connected to.
+
+``value``
+    The network name. This can be an ssid or any other identifier but should be
+    unique.
+``latitude``
+    Latitude of the networks location.
+``longitude``
+    Longitude of the networks location.
+
+Accounts
+--------
+
+A users account or profile on a webservice, like github or instagram.
+
+``service``
+    The identifier of the service/website. It's recommended to use the websites
+    domain for this as defined in `Domains`_.
+``username``
+    The users unique identifier, like the login name. If the login name is not
+    known or the system doesn't use login names, use the email address instead.
+``displayname``
+    The users display name. This name is often not unique and may contain the
+    users real name.
+``email``
+    The email address associated with the account.
+``url``
+    The url of the public profile if available.
+``last_seen``
+    The last time this account has been active/online.
+
+Breaches
+--------
+
+Either a breach of a specific website, a breach compilation or a breach
+notification service.
+
+``value``
+    The name of the breach, breach compilation or notification service.
+
+Images
+------
+
+``value``
+    The id that identifies the blob. This id is deterministic based on file
+    content.
+``filename``
+    This field is used if we have a well known filename for the content.
+``mime``
+    The image mimetype, like ``image/png`` or ``image/jpeg``.
+``width``
+    The width of the image.
+``height``
+    The height of the image.
+``created``
+    The date and time this image has been taken.
+``latitude``
+    Latitude this picture has been taken.
+``longitude``
+    Longitude this picture has been taken.
+``nudity``
+    A score that classifies nudity in this picture. The score goes from 0 to 2
+    and is commonly calculated with ``img_nudity``. A score above 1 means
+    nudity has been detected.
+``ahash``
+    The Mean (aHash) perceptual hash.
+``dhash``
+    The Gradient (dHash) perceptual hash.
+``phash``
+    The DCT (pHash) perceptual hash.
+
+Relations
+---------
+
+Relations are linking two structs together. The link may contain additional information.
+
+subdomain_ipaddr
+~~~~~~~~~~~~~~~~
+
+Links an ip address to a subdomain.
+
+``subdomain_id``
+    The numeric id of a subdomain struct.
+``ip_addr_id``
+    The numeric id of an ip addr struct.
+
+network_device
+~~~~~~~~~~~~~~
+
+Links a device to a network. This is commonly used with ``db_add_ttl`` so the
+link automatically expires. This is frequently used to monitor networks for
+known and unknown devices.
+
+``network_id``
+    The numeric id of a network struct.
+``device_id``
+    The numeric id of a device struct.
+``ipaddr``
+    The ip address assigned to the device.
+``last_seen``
+    The last time we've seen the device on that network.
+
+breach_email
+~~~~~~~~~~~~
+
+Links an email to a breach. If we know the password as well we can add it to
+the link. If we don't know the password we can leave it blank and fill it
+later. An email can be linked to a breach multiple times with different
+passwords. There is a special upserting logic in place to support this.
+
+``breach_id``
+    The numeric id of a breach struct.
+``email_id``
+    The numeric id of an email struct.
+``password``
+    The password for that email in the breach.

migrations/2019-02-03-123424_accounts/down.sql

@@ -0,0 +1 @@
+DROP TABLE accounts;

migrations/2019-02-03-123424_accounts/up.sql

@@ -0,0 +1,12 @@
+CREATE TABLE accounts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);

migrations/2019-02-11-011316_breaches/down.sql

@@ -0,0 +1,2 @@
+DROP TABLE breach_emails;
+DROP TABLE breaches;

migrations/2019-02-11-011316_breaches/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE breaches (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT breach_unique UNIQUE (value)
+);
+
+CREATE TABLE breach_emails (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    breach_id INTEGER NOT NULL,
+    email_id INTEGER NOT NULL,
+    password VARCHAR,
+    FOREIGN KEY(breach_id) REFERENCES breaches(id) ON DELETE CASCADE,
+    FOREIGN KEY(email_id) REFERENCES emails(id) ON DELETE CASCADE,
+    CONSTRAINT breach_emails_unique UNIQUE (breach_id, email_id, password)
+);

migrations/2019-03-08-060037_images/down.sql

@@ -0,0 +1 @@
+DROP TABLE images;

migrations/2019-03-08-060037_images/up.sql

@@ -0,0 +1,21 @@
+CREATE TABLE images (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+
+    filename VARCHAR,
+    mime VARCHAR,
+    width INT,
+    height INT,
+    created DATETIME,
+
+    latitude FLOAT,
+    longitude FLOAT,
+
+    nudity FLOAT,
+    ahash VARCHAR,
+    dhash VARCHAR,
+    phash VARCHAR,
+
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT image_unique UNIQUE (value)
+);

migrations/2019-03-24-195306_email-names/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _emails_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    CONSTRAINT email_unique UNIQUE (value)
+);
+
+INSERT INTO _emails_new (id, value, unscoped, valid)
+    SELECT id, value, unscoped, valid
+    FROM emails;
+
+DROP TABLE emails;
+ALTER TABLE _emails_new RENAME TO emails;
+
+PRAGMA foreign_keys=on;

migrations/2019-03-24-195306_email-names/up.sql

@@ -0,0 +1,2 @@
+-- Your SQL goes here
+ALTER TABLE emails ADD COLUMN displayname VARCHAR;

modules/dev/arp-scan.lua

@@ -1,5 +1,5 @@
 -- Description: Parse arp-scan output
--- Version: 0.2.0
+-- Version: 0.3.0
 -- License: GPL-3.0
 
 -- sudo arp-scan -qglI wlp3s0

modules/dev/axfr.lua

@@ -4,7 +4,7 @@
 -- License: GPL-3.0
 
 function strip_root_dot(name)
-    m = regex_find("(.+)\\.$", name)
+    local m = regex_find("(.+)\\.$", name)
     if last_err() then return end
 
     if m == nil then
@@ -15,21 +15,19 @@ function strip_root_dot(name)
 end
 
 function add_pointer(name)
-    local domain, domain_id, subdomain_id
-
     -- select psl+1
-    domain = psl_domain_from_dns_name(name)
+    local domain = psl_domain_from_dns_name(name)
     if last_err() then return end
 
     -- add domain
-    domain_id = db_add('domain', {
+    local domain_id = db_add('domain', {
         value=domain,
     })
     if last_err() then return end
     if domain_id == nil then return end
 
     -- add subdomain
-    subdomain_id = db_add('subdomain', {
+    local subdomain_id = db_add('subdomain', {
         domain_id=domain_id,
         value=name,
     })
@@ -37,26 +35,24 @@ function add_pointer(name)
 end
 
 function iter_axfr(zone, arg)
-    local name, r, m, domain
-
     debug(arg)
 
-    name = arg[1]
-    r = arg[2]
+    local name = arg[1]
+    local r = arg[2]
 
     -- select psl+1
-    domain = psl_domain_from_dns_name(name)
+    local domain = psl_domain_from_dns_name(name)
     if last_err() then return end
 
     -- add domain
-    domain_id = db_add('domain', {
+    local domain_id = db_add('domain', {
         value=domain,
     })
     if last_err() then return end
     if domain_id == nil then return end
 
     -- add subdomain
-    subdomain_id = db_add('subdomain', {
+    local subdomain_id = db_add('subdomain', {
         domain_id=domain_id,
         value=name,
     })
@@ -65,7 +61,7 @@ function iter_axfr(zone, arg)
     -- this is a A record
     if r['A'] ~= nil then
         -- add the name and ip
-        ipaddr_id = db_add('ipaddr', {
+        local ipaddr_id = db_add('ipaddr', {
             family='4',
             value=r['A'],
         })
@@ -98,12 +94,10 @@ function iter_axfr(zone, arg)
 end
 
 function iter_a(zone, arg)
-    local i, records, r
-
     if arg == nil then return end
 
     debug('nameserver: ' .. arg)
-    records = dns(zone, {
+    local records = dns(zone, {
         record='AXFR',
         nameserver=arg .. ':53',
         tcp=true,
@@ -112,48 +106,40 @@ function iter_a(zone, arg)
     if records['error'] ~= nil then return end
     records = records['answers']
 
-    i = 1
-    while records[i] ~= nil do
+    for i=1, #records do
         iter_axfr(zone, records[i])
         if last_err() then return end
-        i = i+1
     end
 end
 
 function iter_ns(zone, arg)
-    local i, records, r
-
     if arg == nil then return end
 
-    records = dns(arg, {
+    local records = dns(arg, {
         record='A',
     })
     if last_err() then return end
     if records['error'] ~= nil then return end
     records = records['answers']
 
-    i = 1
-    while records[i] ~= nil do
+    for i=1, #records do
         r = records[i][2]
         iter_a(zone, r['A'])
         if last_err() then return end
-        i = i+1
     end
 end
 
 function run(arg)
-    records = dns(arg['value'], {
+    local records = dns(arg['value'], {
         record='NS',
     })
     if last_err() then return end
     if records['error'] ~= nil then return end
     records = records['answers']
 
-    i = 1
-    while records[i] ~= nil do
-        r = records[i][2]
+    for i=1, #records do
+        local r = records[i][2]
         iter_ns(arg['value'], r['NS'])
         if last_err() then return end
-        i = i+1
     end
 end

modules/dev/cname-harvest.lua

@@ -40,11 +40,9 @@ function run(arg)
     if records['error'] ~= nil then return end
     records = records['answers']
 
-    i = 1
-    while records[i] ~= nil do
+    for i=1, #records do
         r = records[i][2]
         iter(r['CNAME'])
         if last_err() then return end
-        i = i+1
     end
 end

modules/dev/ctlogs.lua

@@ -70,8 +70,7 @@ function run(arg)
 
     seen = {}
 
-    i = 1
-    while i <= #certs do
+    for i=1, #certs do
         c = certs[i]
         debug(c)
 
@@ -92,15 +91,11 @@ function run(arg)
             if last_err() then return end
             names = crt['valid_names']
 
-            j = 1
-            while j <= #names do
+            for j=1, #names do
                 each_name(names[j])
-                j = j+1
             end
         else
             each_name(c['name_value'])
         end
-
-        i = i+1
     end
 end

modules/dev/ddwrt-dhcp.lua

@@ -0,0 +1,88 @@
+-- Description: Export dhcp leases from ddwrt webinterface
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    skip_redacted = not getopt('use-redacted')
+
+    router = getopt('router') -- http://192.0.2.1/
+    if not router then
+        return 'router option is missing (http://192.0.2.1/)'
+    end
+    username = getopt('user')
+    password = getopt('password')
+
+    options = {}
+    if username and password then
+        options['basic_auth'] = {username, password}
+    end
+
+    -- request status page
+    session = http_mksession()
+    url = url_join(router, '/Info.live.htm')
+    req = http_request(session, 'GET', url, options)
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then
+        return 'http error: ' .. resp['status']
+    end
+
+    txt = resp['text']
+    debug(txt)
+
+    -- extract leases from response
+    dhcp_section = regex_find('\\{dhcp_leases:: ([^\\}]+)\\}', txt)
+    if last_err() then return end
+    if not dhcp_section then
+        return 'Failed to get dhcp lease section'
+    end
+
+    leases = regex_find_all('\'([^\']+)\',\'([^\']+)\',\'([^\']+)\',\'[^\']+\',\'[^\']+\'', dhcp_section[2])
+    if last_err() then return end
+
+    now = datetime()
+
+    -- add devices to database
+    for i=1, #leases do
+        local hostname = leases[i][2]
+        local ipaddr = leases[i][3]
+        local macaddr = leases[i][4]
+
+        debug({
+            hostname=hostname,
+            ipaddr=ipaddr,
+            macaddr=macaddr,
+        })
+
+        if skip_redacted and macaddr:match('^xx:xx:') then
+            info('Skipping redacted macaddr')
+        else
+            local device = {
+                value=macaddr,
+                last_seen=now,
+            }
+            if hostname ~= '*' then
+                device['hostname'] = hostname
+            end
+
+            local device_id = db_add('device', device)
+
+            db_add_ttl('network-device', {
+                network_id=network_id,
+                device_id=device_id,
+                ipaddr=ipaddr,
+                last_seen=now,
+            }, 120)
+        end
+    end
+end

modules/dev/dns-ptr.lua

@@ -17,8 +17,7 @@ function run(arg)
         if records['error'] ~= nil then return end
         records = records['answers']
 
-        i = 1
-        while records[i] ~= nil do
+        for i=1, #records do
             r = records[i][2]
             if r['PTR'] then
                 db_update('ipaddr', arg, {
@@ -26,7 +25,6 @@ function run(arg)
                 })
                 if last_err() then return end
             end
-            i = i+1
         end
     end
 end

modules/dev/dns-resolve.lua

@@ -22,8 +22,7 @@ function run(arg)
 
     records = records['answers']
 
-    i = 1
-    while records[i] ~= nil do
+    for i=1, #records do
         r = records[i][2]
         if r['A'] ~= nil then
             ipaddr_id = db_add('ipaddr', {
@@ -38,6 +37,5 @@ function run(arg)
             })
             if last_err() then return end
         end
-        i = i+1
     end
 end

modules/dev/exif.lua

@@ -0,0 +1,12 @@
+-- Description: Extract exif data from images
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    exif = img_exif(arg['value'])
+    if last_err() then return end
+    debug(exif)
+
+    db_update('image', arg, exif)
+end

modules/dev/github.lua

@@ -0,0 +1,107 @@
+-- Description: Collect data from github profiles
+-- Version: 0.1.0
+-- Source: accounts:github.com
+-- License: GPL-3.0
+
+function api_get(url)
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+    -- TODO: set_error(?)
+    if resp['status'] == 403 then return 'ratelimit exceeded' end
+    if resp['status'] ~= 200 then return 'invalid status code' end
+
+    local data = json_decode(resp['text'])
+    if last_err() then return end
+    return data
+end
+
+function import_gpg(url)
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+
+    local key = pgp_pubkey_armored(resp['text'])
+    if not key['uids'] then return end
+
+    for i=1, #key['uids'] do
+        local k = key['uids'][i]
+        debug(k)
+        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", k)
+        if m then
+            db_add('email', {
+                value=m[3],
+                displayname=m[2],
+            })
+        end
+    end
+end
+
+function scan4email(username)
+    local url = 'https://api.github.com/users/' .. username .. '/repos'
+    local repos = api_get(url)
+    if last_err() then return end
+
+    -- XXX: 'https://api.github.com/users/' .. username .. '/events/public?page=0&per_page=100' is faster but less accurate
+
+    for i=1, #repos do
+        local repo = repos[i]
+        debug(repo)
+        local commits = api_get(repo['url'] .. '/commits')
+        if last_err() then return end
+
+        for j=1, #commits do
+            local commit = commits[j]
+            debug(commit)
+
+            if commit['author'] and commit['author']['login'] == username then
+                local name = commit['commit']['author']['name']
+                local email = commit['commit']['author']['email']
+                db_add('email', {
+                    value=email,
+                    displayname=name,
+                })
+                return email
+            end
+
+            if commit['committer'] and commit['committer']['login'] == username then
+                local name = commit['commit']['committer']['name']
+                local email = commit['commit']['committer']['email']
+                db_add('email', {
+                    value=email,
+                    displayname=name,
+                })
+                return email
+            end
+        end
+    end
+end
+
+function run(arg)
+    session = http_mksession()
+    local url = 'https://api.github.com/users/' .. arg['username']
+
+    local data = api_get(url)
+    if last_err() then return end
+    debug(data)
+
+    -- company = data['company']
+    -- location = data['location']
+    -- homepage = data['blog']
+
+    url = 'https://github.com/' .. arg['username'] .. '.gpg'
+    import_gpg(url)
+    if last_err() then return end
+
+    local email = data['email']
+    if not email and not arg['email'] then
+        email = scan4email(arg['username'])
+        if last_err() then return end
+    end
+
+    db_update('account', arg, {
+        url=data['html_url'],
+        displayname=data['name'],
+        email=email,
+    })
+end

modules/dev/hackertarget-subdomains.lua

@@ -18,13 +18,10 @@ function run(arg)
 
     m = regex_find_all("([^,]+),.+\\n?", resp['text'])
 
-    i = 1
-    while i <= #m do
+    for i=1, #m do
         db_add('subdomain', {
             domain_id=arg['id'],
             value=m[i][2]
         })
-
-        i = i+1
     end
 end

modules/dev/home-assistant-devices.lua

@@ -0,0 +1,56 @@
+-- Description: Query device location from home assistant
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Keyring-Access: home-assistant
+
+function run()
+    -- parsing options
+    instance = getopt('instance')
+    if not instance then
+        return 'instance option is missing'
+    end
+
+    host = url_parse(instance)
+    if last_err() then return end
+    host = host['host']
+
+    entity = getopt('entity')
+    if not entity then
+        return 'entity option is missing'
+    end
+
+    -- fetching credentials
+    creds = keyring('home-assistant:' .. host)
+    if creds[1] == nil then
+        profile = url_join(instance, 'profile')
+        return 'missing home-assistant:' .. host .. ' Long-Lived Access Token, open ' .. profile
+    end
+    token = creds[1]['secret_key']
+
+    headers = {}
+    headers['Authorization'] = 'Bearer ' .. token
+    headers['Content-Type'] = 'application/json'
+
+    -- requesting status
+    session = http_mksession()
+    url = url_join(instance, 'api/states/' .. entity)
+    req = http_request(session, 'GET', url, {
+        headers=headers
+    })
+    r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then
+        return 'http error: ' .. r['status']
+    end
+
+    m = json_decode(r['text'])
+    if last_err() then return end
+    debug(m)
+
+    info({
+        gps_accuracy=m['attributes']['gps_accuracy'],
+        longitude=m['attributes']['longitude'],
+        latitude=m['attributes']['latitude'],
+        last_updated=m['last_updated'],
+    })
+end

modules/dev/images.lua

@@ -0,0 +1,9 @@
+-- Description: Parse image metadata
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    local img = img_load(arg['value'])
+    db_update('image', arg, img)
+end

modules/dev/instagram.lua

@@ -0,0 +1,150 @@
+-- Description: Collect data from instagram profiles
+-- Version: 0.1.0
+-- Source: accounts:instagram.com
+-- License: GPL-3.0
+
+PAGE_SIZE = 50
+
+function get_shared_data(html)
+    local s = html_select_list(html, 'script')
+
+    for i=1, #s do
+        local m = regex_find('^window\\._sharedData = (.+);$', s[i]['text'])
+        if m then
+            return json_decode(m[2])
+        end
+    end
+end
+
+function sign_request(rhx, json_params)
+    local magic = rhx .. ':' .. json_params
+    local x_instagram_gis = hex(md5(magic))
+    return x_instagram_gis
+end
+
+function download_image(node)
+    local url = node['display_url']
+    debug(url)
+
+    local req = http_request(session, 'GET', url, {
+        into_blob=true,
+    })
+    local r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
+
+    db_add('image', {
+        value=r['blob'],
+    })
+end
+
+function pull_graphql(page)
+    local end_cursor = page['page_info']['end_cursor']
+
+    for i=1, #page['edges'] do
+        -- shortcode = page['edges'][i]['shortcode']
+        local node = page['edges'][i]['node']
+        node['thumbnail_resources'] = nil
+        node['media_preview'] = nil
+        -- debug(node)
+
+        -- if node['__typename'] == 'GraphImage'
+
+        -- node['dimensions']['height']
+        -- node['dimensions']['width']
+        -- ^ not sure how to get that picture
+
+        -- node['taken_at_timestamp']
+        -- location = node['location']
+
+        local err = download_image(node)
+        if last_err() then return end
+        if err ~= nil then return err end
+
+        todo_posts = todo_posts -1
+        debug('posts left: ' .. todo_posts .. '/' .. total_posts)
+    end
+
+    if page['page_info']['has_next_page'] then
+        debug('requesting next page=' .. end_cursor)
+
+        variables = json_encode({
+            id=user['id'],
+            first=PAGE_SIZE,
+            after=end_cursor
+        })
+
+        local headers = {}
+        headers['X-Instagram-GIS'] = sign_request(rhx_gis, variables)
+
+        local req = http_request(session, 'GET', 'https://www.instagram.com/graphql/query/', {
+            query={
+                query_hash='42323d64886122307be10013ad2dcc44',
+                variables=variables,
+            },
+            headers=headers,
+        })
+        r = http_send(req)
+        if last_err() then return end
+        if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
+
+        x = json_decode(r['text'])
+        if last_err() then return end
+        return pull_graphql(x['data']['user']['edge_owner_to_timeline_media'])
+    end
+end
+
+function run(arg)
+    session = http_mksession()
+    local url = 'https://www.instagram.com/' .. arg['username'] .. '/'
+    local req = http_request(session, 'GET', url, {})
+    local resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'invalid status code' end
+    local html = resp['text']
+
+    local ld = html_select(html, 'script[type="application/ld+json"]')
+    if last_err() then return end
+
+    local ld = json_decode(ld['text'])
+    if last_err() then return end
+    --debug(ld)
+
+    if ld['email'] then
+        db_add('email', {
+            value=ld['email'],
+        })
+    end
+
+    -- homepage=ld['url']
+
+    db_update('account', arg, {
+        displayname=ld['name'],
+        email=ld['email'],
+        url=url,
+    })
+
+    -- download images
+    local sd = get_shared_data(html)
+    if last_err() then return end
+    -- debug(sd)
+
+    rhx_gis = sd['rhx_gis']
+    user = sd['entry_data']['ProfilePage'][1]['graphql']['user']
+
+    -- user['full_name']
+    -- user['id']
+    -- user['is_business_account']
+    -- user['is_private']
+    -- user['is_verified']
+    -- user['has_blocked_viewer']
+    -- user['connected_fb_page']
+    -- user['country_block']
+
+    local page = user['edge_owner_to_timeline_media']
+    total_posts = page['count']
+    todo_posts = total_posts
+
+    -- TODO: fast-update abort if image has been downloaded already
+    return pull_graphql(page)
+end

modules/dev/isc-dhcpd-leases.lua

@@ -1,5 +1,5 @@
 -- Description: Parse isc-dhcpd dhcpd.leases(5)
--- Version: 0.1.0
+-- Version: 0.2.0
 -- License: GPL-3.0
 
 -- cat /var/lib/dhcpd/dhcpd.leases

modules/dev/iw-station-dump.lua

@@ -1,5 +1,5 @@
 -- Description: Parse iw station dump
--- Version: 0.1.0
+-- Version: 0.2.0
 -- License: GPL-3.0
 
 -- iw dev wlan0 station dump

modules/dev/keybase-domains.lua

@@ -0,0 +1,28 @@
+-- Description: Find keybase proofs for domains
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
+        query={
+            domain=arg['value'],
+        }
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    x = json_decode(resp['text'])
+    if last_err() then return end
+    debug(x)
+
+    if x['them'][1] == nil then return end
+    them = x['them'][1]
+
+    db_add('account', {
+        service='keybase.io',
+        username=them['basics']['username'],
+    })
+end

modules/dev/keybase-profiles.lua

@@ -0,0 +1,44 @@
+-- Description: Find keybase proofs for online accounts
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: accounts
+
+function run(arg)
+    service = arg['service']
+    if service == 'twitter.com' then
+        service = 'twitter'
+    elseif service == 'github.com' then
+        service = 'github'
+    elseif service == 'reddit.com' then
+        service = 'reddit'
+    elseif service == 'news.ycombinator.com' then
+        service = 'hackernews'
+    elseif service == 'facebook.com' then
+        service = 'facebook'
+    else
+        return
+    end
+
+    query = {}
+    query[service] = arg['username']
+
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
+        query=query,
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    x = json_decode(resp['text'])
+    if last_err() then return end
+    debug(x)
+
+    if x['them'][1] == nil then return end
+    them = x['them'][1]
+
+    db_add('account', {
+        service='keybase.io',
+        username=them['basics']['username'],
+    })
+end

modules/dev/keybase.lua

@@ -0,0 +1,85 @@
+-- Description: Collect accounts and emails from keybase accounts
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: accounts:keybase.io
+
+function extract_mails(pubkey)
+    for j=1, #pubkey['uids'] do
+        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
+        if m then
+            db_add('email', {
+                value=m[3],
+                displayname=m[2],
+            })
+        end
+    end
+end
+
+function add_domain(dns)
+    local domain = psl_domain_from_dns_name(dns)
+    if last_err() then return end
+
+    local domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    if domain ~= dns then
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=dns,
+        })
+    end
+end
+
+function run(arg)
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
+        query={
+            usernames=arg['username'],
+        }
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    x = json_decode(resp['text'])
+    if last_err() then return end
+    debug(x)
+
+    if x['them'][1] == nil then return end
+    them = x['them'][1]
+
+    -- update keybase profile
+    db_update('account', arg, {
+        displayname=them['profile']['full_name'],
+        url='https://keybase.io/'..arg['username'],
+    })
+
+    -- collect emails
+    pubkey = pgp_pubkey_armored(them['public_keys']['primary']['bundle'])
+    debug(pubkey)
+    extract_mails(pubkey)
+
+    -- collect profiles
+    profiles = them['proofs_summary']['all']
+
+    for i=1, #profiles do
+        profile = profiles[i]
+        debug(profile)
+
+        if
+            profile['proof_type'] == 'generic_web_site' or
+            profile['proof_type'] == 'dns'
+        then
+            add_domain(profile['nametag'])
+        else
+            db_add('account', {
+                service=profile['proof_type'],
+                username=profile['nametag'],
+                url=profile['service_url'],
+            })
+        end
+    end
+end

modules/dev/namechk.lua

@@ -0,0 +1,85 @@
+-- Description: Find accounts by username with namechk.com
+-- Version: 0.1.0
+-- Source: accounts
+-- License: GPL-3.0
+
+function get_services(html)
+    local divs = html_select_list(html, '.service')
+    if last_err() then return end
+
+    local services = {}
+
+    for i=1, #divs do
+        services[i] = divs[i]['attrs']['data-name']
+    end
+
+    return services
+end
+
+function run(arg)
+    -- setup session
+    local session = http_mksession()
+    local req = http_request(session, 'GET', 'https://namechk.com/', {})
+    local resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    local token = html_select(resp['text'], 'input[name="authenticity_token"]')
+    local auth_token = token['attrs']['value']
+
+        local headers = {}
+        headers['X-CSRF-Token'] = authenticity_token
+
+    local services = get_services(resp['text'])
+    debug({
+        auth_token=auth_token,
+        services=services,
+    })
+
+    -- trigger the scan
+    local req = http_request(session, 'POST', 'https://namechk.com/', {
+            headers=headers,
+        form={
+            authenticity_token=auth_token,
+            q=arg['username'],
+        }
+    })
+    local resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+    debug(resp)
+
+    local scan = json_decode(resp['text'])
+    if last_err() then return end
+    local scan_token = scan['valid']
+
+    -- get results
+    for i=1, #services do
+        debug(services[i])
+
+        local req = http_request(session, 'POST', 'https://namechk.com/services/check', {
+            headers=headers,
+            form={
+                token=scan_token,
+                fat=auth_token,
+                service=services[i],
+            }
+        })
+        local resp = http_send(req)
+        if last_err() then return end
+
+        if resp['status'] == 200 then
+            local acc = json_decode(resp['text'])
+            if last_err() then return end
+            debug(acc)
+
+            if acc ~= nil and not acc['available'] and acc['status'] == 'unavailable' then
+                db_add('account', {
+                    service=services[i],
+                    username=arg['username'],
+                    url=acc['callback_url'],
+                })
+            end
+        end
+    end
+end

modules/dev/nudity.lua

@@ -0,0 +1,12 @@
+-- Description: Scan collected images for nudity
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    local nudity = img_nudity(arg['value'])
+    debug(nudity)
+    db_update('image', arg, {
+        nudity=nudity['score'],
+    })
+end

modules/dev/otx-subdomains.lua

@@ -18,15 +18,12 @@ function run(arg)
     if last_err() then return end
     o = o['passive_dns']
 
-    i = 1
-    while o[i] do
+    for i=1, #o do
         x = o[i]
 
         db_add('subdomain', {
             domain_id=arg['id'],
             value=x['hostname'],
         })
-
-        i = i+1
     end
 end

modules/dev/passive-spider.lua

@@ -52,12 +52,9 @@ function run(arg)
     end
 
     -- process html links
-    i = 1
-    while i <= #links do
+    for i=1, #links do
         href = links[i]['attrs']['href']
 
         entry(arg['value'], href)
-
-        i = i+1
     end
 end

modules/dev/pgp-keyserver.lua

@@ -20,8 +20,7 @@ function run(arg)
     if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
 
     links = html_select_list(resp['text'], 'a')
-    i = 1
-    while i <= #links do
+    for i=1, #links do
         href = links[i]['attrs']['href']
 
         if href:find('/pks/lookup%?op=get&search=') == 1 then
@@ -40,19 +39,16 @@ function run(arg)
 
             -- TODO: ensure at least one email matches our target domain
             if pubkey['uids'] then
-                j = 1
-                while j <= #pubkey['uids'] do
-                    m = regex_find("<([^< ]+@[^< ]+)>$", pubkey['uids'][j])
+                for j=1, #pubkey['uids'] do
+                    local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
                     if m then
                         db_add('email', {
-                            value=m[2],
+                            value=m[3],
+                            displayname=m[2],
                         })
                     end
-                    j = j+1
                 end
             end
         end
-
-        i = i+1
     end
 end

modules/dev/phpmyadmin.lua

@@ -76,8 +76,7 @@ function run(arg)
 
     session = http_mksession()
 
-    i = 1
-    while i <= #paths do
+    for i=1, #paths do
         p = paths[i]
         url = url_join(arg['value'], p)
         debug(url)
@@ -100,7 +99,5 @@ function run(arg)
                 })
             end
         end
-
-        i = i+1
     end
 end

modules/dev/smtp-check.lua

@@ -14,14 +14,12 @@ function find_mx(domain)
     records = records['answers']
     -- debug(records)
 
-    i = 1
-    while i <= #records do
+    for i=1, #records do
         r = records[i][2]['MX']
         if r then
             debug('mx: ' .. r[2])
             return r[2]
         end
-        i = i+1
     end
 end
 

modules/dev/threatminer-ipaddr.lua

@@ -23,8 +23,7 @@ function run(arg)
     if last_err() then return end
     o = o['results']
 
-    i = 1
-    while o[i] do
+    for i=1, #o do
         x = o[i]
 
         domain = psl_domain_from_dns_name(x['domain'])
@@ -46,7 +45,5 @@ function run(arg)
                 ip_addr_id=arg['id'],
             })
         end
-
-        i = i+1
     end
 end

modules/dev/threatminer-subdomains.lua

@@ -21,15 +21,12 @@ function run(arg)
     if last_err() then return end
     o = o['results']
 
-    i = 1
-    while o[i] do
+    for i=1, #o do
         x = o[i]
 
         db_add('subdomain', {
             domain_id=arg['id'],
             value=x,
         })
-
-        i = i+1
     end
 end

modules/dev/thunderbird-autoconfig.lua

@@ -27,8 +27,7 @@ function run(arg)
 
     m = regex_find_all('<hostname>([^<]+)</hostname>', resp['text'])
 
-    i = 1
-    while i <= #m do
+    for i=1, #m do
         subdomain = m[i][2]
 
         domain = psl_domain_from_dns_name(subdomain)
@@ -42,7 +41,5 @@ function run(arg)
             value=subdomain,
         })
         if last_err() then return end
-
-        i = i+1
     end
 end

modules/dev/url-scan.lua

@@ -1,5 +1,5 @@
 -- Description: Scan subdomains for websites
--- Version: 0.2.0
+-- Version: 0.3.0
 -- Source: subdomains
 -- License: GPL-3.0
 

modules/dev/waybackurls.lua

@@ -29,8 +29,7 @@ function run(arg)
 
     seen = {}
 
-    i = 2
-    while o[i] do
+    for i=2, #o do
         url = o[i][3]
         debug(url)
         parts = url_parse(url)
@@ -51,7 +50,5 @@ function run(arg)
                 seen[subdomain] = 1
             end
         end
-
-        i = i+1
     end
 end

modules/dev/well-known-uris.lua

@@ -24,8 +24,7 @@ function run(arg)
 
     session = http_mksession()
 
-    i = 1
-    while i <= #locations do
+    for i=1, #locations do
         path = locations[i]['path']
         expect_redirect = locations[i]['redirect']
 
@@ -58,7 +57,5 @@ function run(arg)
                 db_add('url', obj)
             end
         end
-
-        i = i+1
     end
 end

modules/harness/tux-img.lua

@@ -0,0 +1,21 @@
+-- Description: Download an image
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+
+    req = http_request(session, 'GET', 'https://www.kernel.org/theme/images/logos/tux.png', {
+        into_blob=true,
+    })
+    r = http_send(req)
+    if last_err() then return end
+    if r['status'] ~= 200 then
+        return 'http error: ' .. r['status']
+    end
+
+    debug(r)
+    db_add('image', {
+        value=r['blob'],
+    })
+end

sn0int-registry/Cargo.toml

@@ -8,9 +8,9 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.4.0", path="sn0int-common" }
-rocket = "0.4"
-rocket_failure = { version = "0.1", features = ["with-rocket"] }
+sn0int-common = { version="0.6.0", path="sn0int-common" }
+rocket = { version = "0.4", default-features=false }
+rocket_failure = { version = "0.1.2", features = ["with-rocket"] }
 rocket_contrib = { version = "0.4", features = ["handlebars_templates"] }
 
 diesel = { version = "1.3", features = ["postgres", "r2d2"] }
@@ -23,7 +23,7 @@ failure = "0.1"
 url = "1.0"
 log = "0.4"
 semver = "0.9.0"
-lazy_static = "1.1"
+lazy_static = "1"
 blake2 = "0.8.0"
 hex = "0.3.1"
 maplit = "1.0.1"
@@ -33,3 +33,7 @@ serde_derive = "1.0"
 serde_json = "1.0"
 
 dotenv = "0.13"
+
+[patch.crates-io]
+rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }
+rocket_contrib = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }

sn0int-registry/migrations/2019-03-02-130236_publish-time/down.sql

@@ -0,0 +1 @@
+ALTER TABLE releases DROP COLUMN published;

sn0int-registry/migrations/2019-03-02-130236_publish-time/up.sql

@@ -0,0 +1 @@
+ALTER TABLE releases ADD COLUMN published TIMESTAMP NOT NULL DEFAULT NOW();

sn0int-registry/sn0int-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.4.0"
+version = "0.6.0"
 description = "Common code for sn0int"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"

sn0int-registry/sn0int-common/src/api.rs

@@ -46,3 +46,8 @@ impl SearchResponse {
         format!("{}/{}", self.author, self.name)
     }
 }
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct LatestResponse {
+    pub time: Option<u64>,
+}

sn0int-registry/sn0int-common/src/metadata.rs

@@ -35,6 +35,11 @@ pub enum Source {
     Urls,
     Emails,
     PhoneNumbers,
+    Networks,
+    Devices,
+    Accounts(Option<String>),
+    Breaches,
+    Images,
     KeyRing(String),
 }
 
@@ -56,6 +61,11 @@ impl FromStr for Source {
             ("urls", None) => Ok(Source::Urls),
             ("emails", None) => Ok(Source::Emails),
             ("phonenumbers", None) => Ok(Source::PhoneNumbers),
+            ("networks", None) => Ok(Source::Networks),
+            ("devices", None) => Ok(Source::Devices),
+            ("accounts", param) => Ok(Source::Accounts(param.map(String::from))),
+            ("breaches", None) => Ok(Source::Breaches),
+            ("images", None) => Ok(Source::Images),
             ("keyring", Some(param)) => Ok(Source::KeyRing(param.to_string())),
             (x, Some(param)) => bail!("Unknown Source: {:?} ({:?})", x, param),
             (x, None) => bail!("Unknown Source: {:?}", x),
@@ -244,4 +254,13 @@ mod tests {
         let x = Source::from_str("keyring");
         assert!(x.is_err());
     }
+
+    #[test]
+    fn verify_account_source() {
+        let x = Source::from_str("accounts").unwrap();
+        assert_eq!(x, Source::Accounts(None));
+
+        let x = Source::from_str("accounts:github.com").unwrap();
+        assert_eq!(x, Source::Accounts(Some("github.com".into())));
+    }
 }

sn0int-registry/src/auth.rs

@@ -2,7 +2,7 @@ use crate::errors::*;
 use diesel::pg::PgConnection;
 use oauth2::basic::BasicClient;
 use oauth2::prelude::*;
-use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl};
+use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl, TokenResponse};
 use crate::github::GithubAuthenticator;
 use crate::models::AuthToken;
 use url::Url;

sn0int-registry/src/main.rs

@@ -78,12 +78,17 @@ fn run() -> Result<()> {
         }))
         .mount("/api/v0", routes![
             routes::api::quickstart,
+            routes::api::latest,
             routes::api::search,
             routes::api::info,
             routes::api::download,
             routes::api::publish,
             routes::api::whoami,
         ])
+        .mount("/api/v0/badges", routes![
+            routes::badges::modules,
+            routes::badges::downloads,
+        ])
         .mount("/auth", routes![
             routes::auth::get,
             routes::auth::post,
@@ -93,6 +98,7 @@ fn run() -> Result<()> {
             routes::assets::index,
             routes::assets::favicon,
             routes::assets::style,
+            routes::health::health,
         ])
         .register(catchers![
             bad_request,

sn0int-registry/src/models.rs

@@ -4,6 +4,7 @@ use diesel::pg::PgConnection;
 use diesel::sql_types::BigInt;
 use diesel_full_text_search::{plainto_tsquery, TsQueryExtensions};
 use crate::schema::*;
+use std::time::SystemTime;
 
 
 #[derive(AsChangeset, Serialize, Deserialize, Queryable, Insertable)]
@@ -159,7 +160,7 @@ impl Module {
                 modules::description,
                 modules::latest,
                 modules::featured,
-                diesel::dsl::sql::<BigInt>("sum(releases.downloads) AS sum"),
+                diesel::dsl::sql::<BigInt>("coalesce(sum(releases.downloads), 0) AS sum"),
             ))
             .left_join(releases::table)
             .group_by(modules::id)
@@ -194,6 +195,15 @@ impl Module {
             .load(connection)
             .map_err(Error::from)
     }
+
+    pub fn count(connection: &PgConnection) -> Result<i64> {
+        use diesel::dsl::*;
+
+        modules::table
+            .select(count(modules::id))
+            .first(connection)
+            .map_err(Error::from)
+    }
 }
 
 #[derive(Insertable)]
@@ -214,6 +224,7 @@ pub struct Release {
     pub version: String,
     pub downloads: i32,
     pub code: String,
+    pub published: SystemTime,
 }
 
 impl Release {
@@ -271,6 +282,21 @@ impl Release {
             .execute(connection)?;
         Ok(())
     }
+
+    pub fn latest(connection: &PgConnection) -> Result<Option<Release>> {
+        releases::table
+            .order_by(releases::published.desc())
+            .first::<Release>(connection)
+            .optional()
+            .map_err(Error::from)
+    }
+
+    pub fn downloads(connection: &PgConnection) -> Result<i64> {
+        releases::table
+            .select(diesel::dsl::sql::<BigInt>("coalesce(sum(releases.downloads), 0) AS sum"))
+            .first(connection)
+            .map_err(Error::from)
+    }
 }
 
 #[derive(Insertable)]

sn0int-registry/src/routes/api.rs

@@ -9,6 +9,7 @@ use semver::Version;
 use sn0int_common::api::*;
 use sn0int_common::id;
 use sn0int_common::metadata::Metadata;
+use std::time::UNIX_EPOCH;
 
 
 #[get("/quickstart")]
@@ -17,6 +18,20 @@ pub fn quickstart(connection: db::Connection) -> ApiResult<ApiResponse<Vec<Modul
     Ok(ApiResponse::Success(modules))
 }
 
+#[get("/latest")]
+pub fn latest(connection: db::Connection) -> ApiResult<ApiResponse<LatestResponse>> {
+    let time = Release::latest(&connection)?
+        .map(|x| {
+            x.published.duration_since(UNIX_EPOCH)
+                .expect("Time went backwards")
+                .as_secs()
+        });
+    let latest = LatestResponse {
+        time,
+    };
+    Ok(ApiResponse::Success(latest))
+}
+
 #[derive(Debug, FromForm)]
 pub struct Search {
     q: String,

sn0int-registry/src/routes/badges.rs

@@ -0,0 +1,45 @@
+use crate::errors::*;
+use crate::db;
+use crate::models::*;
+use rocket_contrib::json::Json;
+
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Badge {
+    schema_version: u8,
+    label: String,
+    message: String,
+    color: &'static str,
+}
+
+impl Badge {
+    pub fn new(label: String, message: String) -> Badge {
+        Badge {
+            schema_version: 1,
+            label,
+            message,
+            color: "blue",
+        }
+    }
+}
+
+#[get("/modules")]
+pub fn modules(connection: db::Connection) -> ApiResult<Json<Badge>> {
+    let num = Module::count(&connection)?;
+    Ok(Json(Badge::new("modules".into(), num.to_string())))
+}
+
+#[get("/downloads")]
+pub fn downloads(connection: db::Connection) -> ApiResult<Json<Badge>> {
+    let mut num = Release::downloads(&connection)?;
+
+    let mut unit = String::new();
+    while num > 1000 {
+        num /= 1000;
+        unit.push('k');
+    }
+
+    let message = format!("{:.1}{}", num, unit);
+    Ok(Json(Badge::new("downloads".into(), message)))
+}

sn0int-registry/src/routes/health.rs

@@ -0,0 +1,8 @@
+use crate::errors::*;
+use crate::db;
+
+
+#[get("/health")]
+pub fn health(_connection: db::Connection) -> ApiResult<()> {
+    Ok(())
+}

sn0int-registry/src/routes/mod.rs

@@ -1,3 +1,5 @@
 pub mod api;
 pub mod assets;
 pub mod auth;
+pub mod badges;
+pub mod health;

sn0int-registry/src/schema.rs

@@ -34,6 +34,7 @@ table! {
         version -> Varchar,
         downloads -> Int4,
         code -> Text,
+        published -> Timestamp,
     }
 }
 

src/api.rs

@@ -46,6 +46,7 @@ impl Client {
             info!("Adding session token to request");
             request.header("Auth", session.as_str());
         }
+        request.header("User-Agent", web::default_user_agent());
 
         let request = request.body(body)?;
 
@@ -131,4 +132,10 @@ impl Client {
         let reply = self.get::<Vec<ModuleInfoResponse>>(&url)?;
         Ok(reply)
     }
+
+    pub fn latest_release(&self) -> Result<LatestResponse> {
+        let url = format!("{}/api/v0/latest", self.server);
+        let reply = self.get::<LatestResponse>(&url)?;
+        Ok(reply)
+    }
 }

src/args.rs

@@ -9,8 +9,8 @@ use crate::workspaces::Workspace;
 #[structopt(author = "",
             raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
-    #[structopt(short="w", long="workspace")]
     /// Select a different workspace instead of the default
+    #[structopt(short="w", long="workspace")]
     pub workspace: Option<Workspace>,
 
     #[structopt(subcommand)]
@@ -28,66 +28,75 @@ impl Args {
 
 #[derive(Debug, StructOpt)]
 pub enum SubCommand {
-    #[structopt(author="", name="run")]
     /// Run a module directly
+    #[structopt(author="", name="run")]
     Run(Run),
-    #[structopt(author="", name="sandbox")]
     /// For internal use
+    #[structopt(author="", name="sandbox")]
     Sandbox(Sandbox),
-    #[structopt(author="", name="login")]
     /// Login to the registry for publishing
+    #[structopt(author="", name="login")]
     Login(Login),
-    #[structopt(author="", name="new")]
     /// Create a new module
+    #[structopt(author="", name="new")]
     New(New),
-    #[structopt(author="", name="publish")]
     /// Publish a script to the registry
+    #[structopt(author="", name="publish")]
     Publish(Publish),
-    #[structopt(author="", name="install")]
     /// Install a module from the registry
+    #[structopt(author="", name="install")]
     Install(Install),
-    #[structopt(author="", name="search")]
     /// Search in the registry
+    #[structopt(author="", name="search")]
     Search(Search),
-    #[structopt(author="", name="select")]
+    /// Insert into the database
+    #[structopt(author="", name="add")]
+    Add(cmd::add_cmd::Args),
     /// Select from the database
+    #[structopt(author="", name="select")]
     Select(cmd::select_cmd::Args),
-    #[structopt(author="", name="completions")]
+    /// Delete from the database
+    #[structopt(author="", name="delete")]
+    Delete(cmd::delete_cmd::Args),
+    /// Verify blob storage for corrupt and dangling blobs
+    #[structopt(author="", name="fsck")]
+    Fsck(cmd::fsck_cmd::Args),
     /// Generate shell completions
+    #[structopt(author="", name="completions")]
     Completions(Completions),
 }
 
 #[derive(Debug, StructOpt)]
 pub struct Run {
     /// Execute a module that has been installed
-    pub module: Option<String>,
-    #[structopt(short="f", long="file", conflicts_with="module")]
+    pub module: String,
     /// Run a module from a path
-    pub file: Option<String>,
-    #[structopt(short="j", long="threads", default_value="1")]
+    #[structopt(short="f", long="file")]
+    pub file: bool,
     /// Run modules concurrently
+    #[structopt(short="j", long="threads", default_value="1")]
     pub threads: usize,
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
     /// Verbose logging, once to print inserts even if they don't add new
     /// data, twice to activate the debug() function
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
     pub verbose: u64,
-    #[structopt(long="stdin")]
     /// Expose stdin to modules
+    #[structopt(long="stdin")]
     pub stdin: bool,
-    #[structopt(long="grant")]
     /// Automatically grant access to a keyring namespace
+    #[structopt(long="grant")]
     pub grants: Vec<String>,
-    #[structopt(long="grant-full-keyring")]
     /// Automatically grant access to all requested keys
+    #[structopt(long="grant-full-keyring")]
     pub grant_full_keyring: bool,
-    #[structopt(long="deny-keyring")]
     /// Automatically deny access to all requested keys
+    #[structopt(long="deny-keyring")]
     pub deny_keyring: bool,
-    #[structopt(short="x", long="exit-on-error")]
     /// Exit on first error and set exit code
+    #[structopt(short="x", long="exit-on-error")]
     pub exit_on_error: bool,
-    #[structopt(short="o", long="option")]
     /// Set an option
+    #[structopt(short="o", long="option")]
     pub options: Vec<options::Opt>,
 }
 

src/auth.rs

@@ -1,6 +1,7 @@
 use crate::errors::*;
 use opener;
 use std::fs;
+use std::path::PathBuf;
 use std::thread;
 use std::time::Duration;
 use crate::api::Client;
@@ -9,17 +10,18 @@ use crate::paths;
 use crate::term;
 
 
-pub fn load_token() -> Result<String> {
+fn path() -> Result<PathBuf> {
     let path = paths::data_dir()?;
-    let path = path.join("auth");
-    let session = fs::read_to_string(path)?;
+    Ok(path.join("auth"))
+}
+
+pub fn load_token() -> Result<String> {
+    let session = fs::read_to_string(path()?)?;
     Ok(session.trim().to_string())
 }
 
 pub fn save_token(session: &str) -> Result<()> {
-    let path = paths::data_dir()?;
-    let path = path.join("auth");
-    fs::write(path, format!("{}\n", session))?;
+    fs::write(path()?, format!("{}\n", session))?;
     Ok(())
 }
 

src/blobs.rs

@@ -0,0 +1,231 @@
+use crate::errors::*;
+use crate::paths;
+use crate::worker::{EventWithCallback, Event2};
+use crate::workspaces::Workspace;
+
+use blake2::VarBlake2b;
+use digest::{Input, VariableOutput};
+use bytes::Bytes;
+use serde::ser::{Serialize, Serializer};
+use serde::de::{self, Deserialize, Deserializer};
+use std::fs;
+use std::path::PathBuf;
+use std::result;
+use std::sync::mpsc;
+
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Blob {
+    pub id: String,
+    pub bytes: Bytes,
+}
+
+impl Blob {
+    pub fn create(bytes: Bytes) -> Blob {
+        let id = Self::hash(&bytes);
+        Blob {
+            id,
+            bytes,
+        }
+    }
+
+    pub fn hash(bytes: &[u8]) -> String {
+        let mut h = VarBlake2b::new(32).unwrap();
+        h.input(bytes);
+        Self::encode_hash(&h.vec_result())
+    }
+
+    #[inline]
+    fn encode_hash(bytes: &[u8]) -> String {
+        let x = bs58::encode(bytes).into_string();
+        format!("{:0<44}", x)
+    }
+}
+
+impl EventWithCallback for Blob {
+    type Payload = ();
+
+    fn with_callback(self, tx: mpsc::Sender<result::Result<Self::Payload, String>>) -> Event2 {
+        Event2::Blob((self, tx))
+    }
+}
+
+impl Serialize for Blob {
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let s = base64::encode(&self.bytes);
+        serializer.serialize_str(&s)
+    }
+}
+
+impl<'de> Deserialize<'de> for Blob {
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let bytes = base64::decode(&s)
+            .map_err(de::Error::custom)?;
+        Ok(Blob::create(Bytes::from(bytes)))
+    }
+}
+
+pub struct BlobStorage {
+    path: PathBuf,
+}
+
+impl BlobStorage {
+    #[inline]
+    pub fn new<I: Into<PathBuf>>(path: I) -> BlobStorage {
+        BlobStorage {
+            path: path.into(),
+        }
+    }
+
+    pub fn workspace(workspace: &Workspace) -> Result<BlobStorage> {
+        let path = paths::blobs_dir(workspace)?;
+        Ok(BlobStorage::new(path))
+    }
+
+    pub fn join(&self, id: &str) -> Result<PathBuf> {
+        if !id.chars().all(char::is_alphanumeric) {
+            bail!("blob id contains invalid characters");
+        }
+        Ok(self.path.join(id))
+    }
+
+    pub fn load(&self, id: &str) -> Result<Blob> {
+        let path = self.join(id)?;
+
+        debug!("Loading blob from {:?}", path);
+        let bytes = fs::read(path)
+            .context("Failed to read blob")?;
+
+        Ok(Blob {
+            id: id.to_string(),
+            bytes: Bytes::from(bytes),
+        })
+    }
+
+    pub fn save(&self, blob: &Blob) -> Result<()> {
+        let path = self.join(&blob.id)?;
+
+        debug!("Writing blob to {:?}", path);
+        fs::write(path, &blob.bytes)
+            .context("Failed to write blob")?;
+
+        Ok(())
+    }
+
+    pub fn delete(&self, id: &str) -> Result<()> {
+        let path = self.join(id)?;
+        debug!("Deleting blob: {:?}", path);
+        fs::remove_file(path)
+            .context("Failed to delete blob")?;
+        Ok(())
+    }
+
+    pub fn list(&self) -> Result<Vec<String>> {
+        let mut blobs = Vec::new();
+        for entry in fs::read_dir(&self.path)? {
+            let blob = entry?
+                .file_name()
+                .into_string()
+                .map_err(|_| format_err!("Invalid filename"))?;
+            blobs.push(blob);
+        }
+        Ok(blobs)
+    }
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile;
+    use serde_json;
+
+    #[inline]
+    fn blob() -> (Bytes, Blob) {
+        let bytes = Bytes::from(&b"asdf"[..]);
+        (bytes.clone(), Blob::create(bytes))
+    }
+
+    #[test]
+    fn verify_create_blob() {
+        let (bytes, blob) = blob();
+        assert_eq!(blob, Blob {
+            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
+            bytes,
+        });
+    }
+
+    #[test]
+    fn test_blobstorage_save() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let s = BlobStorage::new(dir.path());
+
+        let (_, blob1) = blob();
+        s.save(&blob1).expect("save failed");
+
+        let blob2 = s.load("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz").expect("load failed");
+
+        assert_eq!(blob1, blob2);
+    }
+
+    #[test]
+    fn test_blobstorage_load_failure() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let s = BlobStorage::new(dir.path());
+
+        let result = s.load("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz");
+
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_path_validation() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let s = BlobStorage::new(dir.path());
+
+        let result = s.load("../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../etc/passwd");
+
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_serialize() {
+        let (_, blob) = blob();
+        let json = serde_json::to_string(&blob).unwrap();
+        assert_eq!(&json, "\"YXNkZg==\"");
+    }
+
+    #[test]
+    fn test_deserialize() {
+        let (_, blob1) = blob();
+        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
+        assert_eq!(blob1, blob2);
+    }
+
+    #[test]
+    fn test_hash_encoding() {
+        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
+    }
+
+    #[test]
+    fn test_hash_encoding_padding() {
+        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
+    }
+}

src/cmd/add_cmd.rs

@@ -1,28 +1,92 @@
 use crate::errors::*;
 
+use crate::blobs::Blob;
+use crate::cmd::Cmd;
+use crate::gfx;
 use crate::models::*;
 use crate::shell::Readline;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::utils;
+use crate::term;
+use std::fs;
+use std::path::Path;
+use walkdir::WalkDir;
 
 
 #[derive(Debug, StructOpt)]
 #[structopt(author = "",
             raw(global_settings = "&[AppSettings::ColoredHelp]"))]
-pub enum Args {
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Target,
+    /// Do not actually insert into database
+    #[structopt(short="n", long="dry-run")]
+    dry_run: bool,
+}
+
+#[derive(Debug, StructOpt)]
+pub enum Target {
+    /// Insert domain into the database
     #[structopt(name="domain")]
     Domain(AddDomain),
+    /// Insert subdomain into the database
     #[structopt(name="subdomain")]
     Subdomain(AddSubdomain),
+    /// Insert email into the database
     #[structopt(name="email")]
     Email(AddEmail),
+    /// Insert phonenumber into the database
     #[structopt(name="phonenumber")]
     PhoneNumber(AddPhoneNumber),
+    /// Insert device into the database
     #[structopt(name="device")]
     Device(AddDevice),
+    /// Insert network into the database
     #[structopt(name="network")]
     Network(AddNetwork),
+    /// Insert account into the database
+    #[structopt(name="account")]
+    Account(AddAccount),
+    /// Insert breach into the database
+    #[structopt(name="breach")]
+    Breach(AddBreach),
+    /// Insert images into the database
+    #[structopt(name="image")]
+    Image(AddImage),
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Readline) -> Result<()> {
+        match self.subcommand {
+            Target::Domain(args) => args.insert(rl, self.dry_run),
+            Target::Subdomain(args) => args.insert(rl, self.dry_run),
+            Target::Email(args) => args.insert(rl, self.dry_run),
+            Target::PhoneNumber(args) => args.insert(rl, self.dry_run),
+            Target::Device(args) => args.insert(rl, self.dry_run),
+            Target::Network(args) => args.insert(rl, self.dry_run),
+            Target::Account(args) => args.insert(rl, self.dry_run),
+            Target::Breach(args) => args.insert(rl, self.dry_run),
+            Target::Image(args) => args.insert(rl, self.dry_run),
+        }
+    }
+}
+
+#[inline]
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    Args::run_str(rl, args)
+}
+
+trait IntoInsert: Sized {
+    fn into_insert(self, rl: &Readline) -> Result<Insert>;
+
+    fn insert(self, rl: &Readline, dry_run: bool) -> Result<()> {
+        let insert = self.into_insert(rl)?;
+        if !dry_run {
+            rl.db().insert_generic(insert)?;
+        }
+        Ok(())
+    }
 }
 
 #[derive(Debug, StructOpt)]
@@ -30,28 +94,143 @@ pub struct AddDomain {
     domain: Option<String>,
 }
 
+impl IntoInsert for AddDomain {
+    fn into_insert(self, rl: &Readline) -> Result<Insert> {
+        let domain = match self.domain {
+            Some(domain) => domain,
+            _ => utils::question("Domain")?,
+        };
+
+        // ensure input is a valid domain
+        let parsed_domain = rl.psl().parse_domain(&domain)
+            .map_err(|e| format_err!("Failed to parse domain: {}", e))?;
+
+        if Some(domain.as_str()) != parsed_domain.root() {
+            bail!("This is not a valid domain, might be a subdomain or tld");
+        }
+
+        Ok(Insert::Domain(NewDomain {
+            value: domain,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddSubdomain {
     subdomain: Option<String>,
 }
 
+impl IntoInsert for AddSubdomain {
+    fn into_insert(self, rl: &Readline) -> Result<Insert> {
+        let subdomain = match self.subdomain {
+            Some(subdomain) => subdomain,
+            _ => utils::question("Subdomain")?,
+        };
+
+        let dns_name = rl.psl().parse_dns_name(&subdomain)
+            .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
+
+        let domain = dns_name.domain()
+            .ok_or_else(|| format_err!("Dns Name seems invalid"))?
+            .to_string();
+
+        let domain_id = match rl.db().insert_struct(NewDomain {
+            value: domain,
+        })? {
+            Some((_, domain_id)) => domain_id,
+            _ => bail!("Domain is out out of scope"),
+        };
+
+        Ok(Insert::Subdomain(NewSubdomain {
+            domain_id,
+            value: subdomain,
+            resolvable: None,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddEmail {
     email: Option<String>,
 }
 
+impl IntoInsert for AddEmail {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let email = match self.email {
+            Some(email) => email,
+            _ => utils::question("Email")?,
+        };
+
+        Ok(Insert::Email(NewEmail {
+            value: email,
+            displayname: None,
+            valid: None,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddPhoneNumber {
     phonenumber: Option<String>,
     name: Option<String>,
 }
 
+impl IntoInsert for AddPhoneNumber {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let (phonenumber, name) = match self.phonenumber {
+            Some(phonenumber) => (phonenumber, self.name),
+            _ => {
+                let phonenumber = utils::question("Phone Number")?;
+                let name = utils::question_opt("Name")?;
+                (phonenumber, name)
+            },
+        };
+
+        Ok(Insert::PhoneNumber(NewPhoneNumber {
+            value: phonenumber,
+            name: name,
+            valid: None,
+            last_online: None,
+            country: None,
+            carrier: None,
+            line: None,
+            is_ported: None,
+            last_ported: None,
+            caller_name: None,
+            caller_type: None,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddDevice {
     mac: Option<String>,
     name: Option<String>,
 }
 
+impl IntoInsert for AddDevice {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let (mac, name) = match self.mac {
+            Some(mac) => {
+                (mac, self.name)
+            },
+            _ => {
+                let mac = utils::question("Mac address")?;
+                let name = utils::question_opt("Name")?;
+                (mac, name)
+            },
+        };
+
+        Ok(Insert::Device(NewDevice {
+            value: mac,
+            name: name,
+            hostname: None,
+            vendor: None,
+            last_seen: None,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddNetwork {
     network: Option<String>,
@@ -59,150 +238,177 @@ pub struct AddNetwork {
     longitude: Option<f32>,
 }
 
-pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    match args {
-        Args::Domain(args) => add_domain(rl, args),
-        Args::Subdomain(args) => add_subdomain(rl, args),
-        Args::Email(args) => add_email(rl, args),
-        Args::PhoneNumber(args) => add_phonenumber(rl, args),
-        Args::Device(args) => add_device(rl, args),
-        Args::Network(args) => add_network(rl, args),
+impl IntoInsert for AddNetwork {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let (network, latitude, longitude) = match self.network {
+            Some(network) => (network, self.latitude, self.longitude),
+            _ => {
+                let network = utils::question("Network")?;
+                let latitude = utils::question_typed_opt("Latitude")?;
+                let longitude = utils::question_typed_opt("Longitude")?;
+                (network, latitude, longitude)
+            }
+        };
+
+        Ok(Insert::Network(NewNetwork {
+            value: network,
+            latitude,
+            longitude,
+        }))
     }
 }
 
-fn add_domain(rl: &mut Readline, args: AddDomain) -> Result<()> {
-    let domain = match args.domain {
-        Some(domain) => domain,
-        _ => utils::question("Domain")?,
-    };
-
-    // ensure input is a valid domain
-    let parsed_domain = rl.psl().parse_domain(&domain)
-        .map_err(|e| format_err!("Failed to parse domain: {}", e))?;
-
-    if Some(domain.as_str()) != parsed_domain.root() {
-        bail!("This is not a valid domain, might be a subdomain or tld");
-    }
-
-    rl.db().insert_struct(NewDomain {
-        value: &domain,
-    })?;
-
-    Ok(())
+#[derive(Debug, StructOpt)]
+pub struct AddAccount {
+    service: Option<String>,
+    username: Option<String>,
 }
 
-fn add_subdomain(rl: &mut Readline, args: AddSubdomain) -> Result<()> {
-    let subdomain = match args.subdomain {
-        Some(subdomain) => subdomain,
-        _ => utils::question("Subdomain")?,
-    };
+impl IntoInsert for AddAccount {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let (service, username) = match (self.service, self.username) {
+            (Some(service), Some(username)) => (service, username),
+            _ => {
+                let service = utils::question("Service")?;
+                let username = utils::question("Username")?;
 
-    let dns_name = rl.psl().parse_dns_name(&subdomain)
-        .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
+                (service, username)
+            },
+        };
 
-    let domain = dns_name.domain()
-        .ok_or_else(|| format_err!("Dns Name seems invalid"))?
-        .to_string();
-
-    let domain_id = match rl.db().insert_struct(NewDomain {
-        value: &domain,
-    })? {
-        Some((_, domain_id)) => domain_id,
-        _ => bail!("Domain is out out of scope"),
-    };
-
-    rl.db().insert_struct(NewSubdomain {
-        domain_id,
-        value: &subdomain,
-        resolvable: None,
-    })?;
-
-    Ok(())
-}
-
-fn add_email(rl: &mut Readline, args: AddEmail) -> Result<()> {
-    let email = match args.email {
-        Some(email) => email,
-        _ => utils::question("Email")?,
-    };
-
-    rl.db().insert_struct(NewEmail {
-        value: &email,
-        valid: None,
-    })?;
-
-    Ok(())
-}
-
-fn add_phonenumber(rl: &mut Readline, args: AddPhoneNumber) -> Result<()> {
-    let (phonenumber, name) = match args.phonenumber {
-        Some(phonenumber) => {
-            (phonenumber, args.name)
-        },
-        _ => {
-            let phonenumber = utils::question("Phone Number")?;
-            let name = utils::question_opt("Name")?;
-            (phonenumber, name)
-        },
-    };
-
-    rl.db().insert_struct(NewPhoneNumber {
-        value: &phonenumber,
-        name: name.as_ref(),
-        valid: None,
-        last_online: None,
-        country: None,
-        carrier: None,
-        line: None,
-        is_ported: None,
-        last_ported: None,
-        caller_name: None,
-        caller_type: None,
-    })?;
-
-    Ok(())
-}
-
-fn add_device(rl: &mut Readline, args: AddDevice) -> Result<()> {
-    let (mac, name) = match args.mac {
-        Some(mac) => {
-            (mac, args.name)
-        },
-        _ => {
-            let mac = utils::question("Mac address")?;
-            let name = utils::question_opt("Name")?;
-            (mac, name)
-        },
-    };
-
-    rl.db().insert_struct(NewDevice {
-        value: &mac,
-        name: name.as_ref(),
-        hostname: None,
-        vendor: None,
-        last_seen: None,
-    })?;
-
-    Ok(())
-}
-
-fn add_network(rl: &mut Readline, args: AddNetwork) -> Result<()> {
-    let (network, latitude, longitude) = match args.network {
-        Some(network) => (network, args.latitude, args.longitude),
-        _ => {
-            let network = utils::question("Network")?;
-            let latitude = utils::question_typed_opt("Latitude")?;
-            let longitude = utils::question_typed_opt("Longitude")?;
-            (network, latitude, longitude)
+        if service.contains('/') {
+            // TODO: avoid duplication
+            bail!("Service field can't contain `/`");
         }
-    };
 
-    rl.db().insert_struct(NewNetwork {
-        value: &network,
-        latitude,
-        longitude,
-    })?;
+        let value = format!("{}/{}", service, username);
 
-    Ok(())
+        Ok(Insert::Account(NewAccount {
+            value: value,
+            service: service,
+            username: username,
+            displayname: None,
+            email: None,
+            url: None,
+            last_seen: None,
+        }))
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AddBreach {
+    name: Option<String>,
+}
+
+impl IntoInsert for AddBreach {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        let name = match self.name {
+            Some(name) => name,
+            _ => {
+                let name = utils::question("Name")?;
+                name
+            }
+        };
+
+        Ok(Insert::Breach(NewBreach {
+            value: name,
+        }))
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AddImage {
+    paths: Vec<String>,
+}
+
+impl IntoInsert for AddImage {
+    fn into_insert(self, _rl: &Readline) -> Result<Insert> {
+        unreachable!()
+    }
+
+    fn insert(self, rl: &Readline, dry_run: bool) -> Result<()> {
+        let paths = if self.paths.is_empty() {
+            let path = utils::question("Path")?;
+            vec![path]
+        } else {
+            self.paths
+        };
+
+        for path in paths {
+            for path in WalkDir::new(path) {
+                let path = match path {
+                    Ok(path) => path,
+                    Err(err) => {
+                        let path = err.path().unwrap_or(Path::new("")).display();
+
+                        let err = err.io_error()
+                            .map(|err| err.to_string())
+                            .unwrap_or_else(|| String::from("walkdir failed"));
+                        term::error(&format!("Failed to access entry {:?}: {}", path, err));
+
+                        continue;
+                    },
+                };
+
+                if path.file_type().is_dir() {
+                    debug!("Traversing into directory: {:?}", path);
+                    continue;
+                }
+
+                debug!("Testing: {:?}", path.path());
+
+                let data = match fs::read(path.path()) {
+                    Ok(data) => data,
+                    Err(err) => {
+                        term::error(&format!("Failed to read {:?}: {}", path, err));
+                        continue;
+                    },
+                };
+
+                // check if image
+                let format = match gfx::guess_format(&data) {
+                    Ok(format) => format.mime().to_string(),
+                    _ => {
+                        debug!("Probably not an image, skipping");
+                        continue;
+                    },
+                };
+                debug!("Detected image format: {:?}", format);
+
+                let blob = Blob::create(data.into());
+                if !dry_run {
+                    rl.blobs().save(&blob)?;
+                }
+                let value = blob.id;
+
+                let filename = path.file_name()
+                    .to_string_lossy()
+                    .to_string();
+
+                term::info(&format!("{} {:?}", value, path.path()));
+
+                if !dry_run {
+                    rl.db().insert_generic(Insert::Image(NewImage {
+                        value,
+
+                        filename: Some(filename),
+                        mime: Some(format),
+                        width: None,
+                        height: None,
+                        created: None,
+
+                        latitude: None,
+                        longitude: None,
+
+                        nudity: None,
+                        ahash: None,
+                        dhash: None,
+                        phash: None,
+                    }))?;
+                }
+            }
+        }
+
+        Ok(())
+    }
 }

src/cmd/delete_cmd.rs

@@ -1,6 +1,7 @@
 use crate::errors::*;
 
-use crate::db;
+use crate::cmd::Cmd;
+use crate::filters::{Target, Filter};
 use crate::shell::Readline;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
@@ -11,52 +12,37 @@ use crate::term;
 #[derive(Debug, StructOpt)]
 #[structopt(author = "",
             raw(global_settings = "&[AppSettings::ColoredHelp]"))]
-pub enum Args {
-    #[structopt(name="domains")]
-    Domains(Filter),
-    #[structopt(name="subdomains")]
-    Subdomains(Filter),
-    #[structopt(name="ipaddrs")]
-    IpAddrs(Filter),
-    #[structopt(name="urls")]
-    Urls(Filter),
-    #[structopt(name="emails")]
-    Emails(Filter),
-    #[structopt(name="phonenumbers")]
-    PhoneNumbers(Filter),
-    #[structopt(name="devices")]
-    Devices(Filter),
-    #[structopt(name="networks")]
-    Networks(Filter),
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Target,
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Filter {
-    args: Vec<String>,
-}
-
-impl Filter {
-    pub fn parse(&self) -> Result<db::Filter> {
-        db::Filter::parse(&self.args)
+impl Cmd for Args {
+    fn run(self, rl: &mut Readline) -> Result<()> {
+        let rows = match &self.subcommand {
+            Target::Domains(filter) => delete::<Domain>(rl, &filter),
+            Target::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
+            Target::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
+            Target::Urls(filter) => delete::<Url>(rl, &filter),
+            Target::Emails(filter) => delete::<Email>(rl, &filter),
+            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
+            Target::Devices(filter) => delete::<Device>(rl, &filter),
+            Target::Networks(filter) => delete::<Network>(rl, &filter),
+            Target::Accounts(filter) => delete::<Account>(rl, &filter),
+            Target::Breaches(filter) => delete::<Breach>(rl, &filter),
+            Target::Images(filter) => delete::<Image>(rl, &filter),
+        }?;
+        term::info(&format!("Deleted {} rows", rows));
+        Ok(())
     }
 }
 
+#[inline]
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    let rows = match args {
-        Args::Domains(filter) => delete::<Domain>(rl, &filter),
-        Args::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
-        Args::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
-        Args::Urls(filter) => delete::<Url>(rl, &filter),
-        Args::Emails(filter) => delete::<Email>(rl, &filter),
-        Args::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
-        Args::Devices(filter) => delete::<Device>(rl, &filter),
-        Args::Networks(filter) => delete::<Network>(rl, &filter),
-    }?;
-    term::info(&format!("Deleted {} rows", rows));
-    Ok(())
+    Args::run_str(rl, args)
 }
 
+#[inline]
 fn delete<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<usize> {
-    rl.db().delete::<T>(&filter.parse()?)
+    T::delete(rl.db(), &filter.parse()?)
 }

src/cmd/fsck_cmd.rs

@@ -0,0 +1,88 @@
+use crate::errors::*;
+
+use crate::blobs::Blob;
+use crate::cmd::Cmd;
+use crate::shell::Readline;
+use crate::term;
+use crate::worker;
+use crate::models::*;
+use std::collections::HashSet;
+use structopt::StructOpt;
+
+
+#[derive(Debug, StructOpt)]
+pub struct Args {
+    /// Verbose output
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
+    verbose: u64,
+    /// Delete only dangling blobs
+    #[structopt(long="gc")]
+    gc: bool,
+    /// Delete dangling and corrupted blobs
+    #[structopt(long="gc-all")]
+    gc_all: bool,
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Readline) -> Result<()> {
+        let blobs = rl.blobs();
+
+        let hashset = worker::spawn_fn("Building reference set...", || {
+            let mut hashset = HashSet::new();
+
+            for x in rl.db().list::<Image>()? {
+                hashset.insert(x.value);
+            }
+
+            Ok(hashset)
+        }, true)?;
+
+        for blob in blobs.list()? {
+
+            let state = worker::spawn_fn(&blob, || {
+                let blob = rl.blobs().load(&blob)?;
+
+                // ensure content matches hash
+                if Blob::hash(&blob.bytes) != blob.id {
+                    Ok(State::Corrupted)
+                // ensure blob is referenced by any table
+                } else if !hashset.contains(&blob.id) {
+                    Ok(State::Dangling)
+                } else {
+                    Ok(State::Valid)
+                }
+            }, true);
+
+            match state {
+                Ok(State::Valid) => {
+                    if self.verbose > 0 {
+                        term::info(&format!("{}... ok", blob));
+                    }
+                },
+                Ok(State::Dangling) => {
+                    term::warn(&format!("{}... dangling", blob));
+                    if self.gc || self.gc_all {
+                        blobs.delete(&blob)?;
+                    }
+                },
+                Ok(State::Corrupted) => {
+                    term::error(&format!("{}... corrupted", blob));
+                    if self.gc_all {
+                        blobs.delete(&blob)?;
+                    }
+                },
+                Err(err) => {
+                    term::error(&format!("{}... {}", blob, err));
+                },
+            }
+        }
+
+        Ok(())
+    }
+}
+
+enum State {
+    Valid,
+    Dangling,
+    Corrupted,
+}

src/cmd/mod.rs

@@ -2,7 +2,7 @@ use crate::errors::*;
 use crate::shell::Readline;
 
 pub trait Cmd: structopt::StructOpt + Sized {
-    fn run(&self, rl: &mut Readline) -> Result<()>;
+    fn run(self, rl: &mut Readline) -> Result<()>;
 
     #[inline]
     fn run_str(rl: &mut Readline, args: &[String]) -> Result<()> {
@@ -13,6 +13,7 @@ pub trait Cmd: structopt::StructOpt + Sized {
 
 pub mod add_cmd;
 pub mod delete_cmd;
+pub mod fsck_cmd;
 pub mod help_cmd;
 pub mod run_cmd;
 pub mod use_cmd;

src/cmd/mod_cmd.rs

@@ -8,6 +8,7 @@ use colored::Colorize;
 use crate::engine::Module;
 use crate::registry;
 use crate::shell::Readline;
+use crate::update::AutoUpdater;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::term;
@@ -24,20 +25,20 @@ pub struct Args {
 
 #[derive(Debug, StructOpt)]
 pub enum SubCommand {
-    #[structopt(author="", name="list")]
     /// List installed modules
+    #[structopt(author="", name="list")]
     List(List),
-    #[structopt(author="", name="install")]
     /// Install module from registry
+    #[structopt(author="", name="install")]
     Install(args::Install),
-    #[structopt(author="", name="search")]
     /// Search modules in registry
+    #[structopt(author="", name="search")]
     Search(args::Search),
-    #[structopt(author="", name="reload")]
     /// Reload modules
+    #[structopt(author="", name="reload")]
     Reload(Reload),
-    #[structopt(author="", name="update")]
     /// Update modules
+    #[structopt(author="", name="update")]
     Update(Update),
 }
 
@@ -54,18 +55,27 @@ pub struct Update {
 }
 
 fn update(client: &Client, config: &Config, module: &Module) -> Result<()> {
+    let name = module.canonical();
     let installed = module.version();
-    let infos = client.query_module(&module.id())?;
+
+    let label = format!("Searching for updates {}", name);
+    let infos = worker::spawn_fn(&label, || {
+        client.query_module(&module.id())
+    }, true)?;
     debug!("Latest version: {:?}", infos);
 
     let latest = infos.latest.ok_or_else(|| format_err!("Module doesn't have any released versions"))?;
 
     if installed != latest {
-        term::info(&format!("Updating {}: {:?} -> {:?}", module.canonical(), installed, latest));
-        registry::run_install(&Install {
-            module: module.id(),
-            version: None,
-        }, &config)?;
+        let label = format!("Updating {}: {:?} -> {:?}", &name, installed, latest);
+        worker::spawn_fn(&label, || {
+            registry::run_install(&Install {
+                module: module.id(),
+                version: None,
+            }, &config)
+        }, true)?;
+
+        term::success(&format!("Updated {}: {:?} -> {:?}", &name, installed, latest));
     }
 
     Ok(())
@@ -105,17 +115,25 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
         SubCommand::Update(_) => {
             let client = Client::new(&config)?;
 
+            let mut success = true;
+
             for module in rl.engine().list() {
-                let name = module.canonical();
-                let label = format!("Searching for updates {}", name);
-
-                let result = worker::spawn_fn(&label, || {
-                    update(&client, &config, &module)
-                }, true);
-
-                if let Err(err) = result {
-                    term::error(&format!("Failed to update {}: {:?}", name, err));
+                if module.is_private() {
+                    debug!("{} is a private module, skipping", module.canonical());
+                    continue;
                 }
+
+                if let Err(err) = update(&client, &config, &module) {
+                    term::error(&format!("Failed to update {}: {}", module.canonical(), err));
+                    success = false;
+                }
+            }
+
+            // TODO: keep a list of outdated packages and remove them after they've been updated
+            if success {
+                let mut autoupdate = AutoUpdater::load()?;
+                autoupdate.all_updated();
+                autoupdate.save()?;
             }
 
             // trigger reload

src/cmd/noscope_cmd.rs

@@ -1,6 +1,6 @@
 use crate::errors::*;
 
-use crate::db;
+use crate::filters::{Target, Filter};
 use crate::shell::Readline;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
@@ -11,46 +11,31 @@ use crate::term;
 #[derive(Debug, StructOpt)]
 #[structopt(author = "",
             raw(global_settings = "&[AppSettings::ColoredHelp]"))]
-pub enum Args {
-    #[structopt(name="domains")]
-    Domains(Filter),
-    #[structopt(name="subdomains")]
-    Subdomains(Filter),
-    #[structopt(name="ipaddrs")]
-    IpAddrs(Filter),
-    #[structopt(name="urls")]
-    Urls(Filter),
-    #[structopt(name="emails")]
-    Emails(Filter),
-    #[structopt(name="phonenumbers")]
-    PhoneNumbers(Filter),
-}
-
-#[derive(Debug, StructOpt)]
-pub struct Filter {
-    args: Vec<String>,
-}
-
-impl Filter {
-    pub fn parse(&self) -> Result<db::Filter> {
-        db::Filter::parse(&self.args)
-    }
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Target,
 }
 
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
-    let rows = match args {
-        Args::Domains(filter) => noscope::<Domain>(rl, &filter),
-        Args::Subdomains(filter) => noscope::<Subdomain>(rl, &filter),
-        Args::IpAddrs(filter) => noscope::<IpAddr>(rl, &filter),
-        Args::Urls(filter) => noscope::<Url>(rl, &filter),
-        Args::Emails(filter) => noscope::<Email>(rl, &filter),
-        Args::PhoneNumbers(filter) => noscope::<PhoneNumber>(rl, &filter),
+    let rows = match args.subcommand {
+        Target::Domains(filter) => noscope::<Domain>(rl, &filter),
+        Target::Subdomains(filter) => noscope::<Subdomain>(rl, &filter),
+        Target::IpAddrs(filter) => noscope::<IpAddr>(rl, &filter),
+        Target::Urls(filter) => noscope::<Url>(rl, &filter),
+        Target::Emails(filter) => noscope::<Email>(rl, &filter),
+        Target::PhoneNumbers(filter) => noscope::<PhoneNumber>(rl, &filter),
+        Target::Devices(filter) => noscope::<Device>(rl, &filter),
+        Target::Networks(filter) => noscope::<Network>(rl, &filter),
+        Target::Accounts(filter) => noscope::<Account>(rl, &filter),
+        Target::Breaches(filter) => noscope::<Breach>(rl, &filter),
+        Target::Images(filter) => noscope::<Image>(rl, &filter),
     }?;
     term::info(&format!("Updated {} rows", rows));
     Ok(())
 }
 
+#[inline]
 fn noscope<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<usize> {
-    rl.db().noscope::<T>(&filter.parse()?)
+    T::noscope(rl.db(), &filter.parse()?)
 }

src/cmd/run_cmd.rs

@@ -1,8 +1,8 @@
 use crate::errors::*;
 
 use crate::args;
-use crate::db::{Database, Filter};
-use crate::db::ttl;
+use crate::blobs::{Blob, BlobStorage};
+use crate::db::{ttl, Filter};
 use crate::engine::Module;
 use crate::models::*;
 use crate::shell::Readline;
@@ -67,16 +67,26 @@ impl From<Args> for Params<'static> {
     }
 }
 
-fn prepare_arg<T: Serialize + Model>(x: T) -> Result<(serde_json::Value, Option<String>)> {
+fn prepare_arg<T: Serialize + Model>(bs: &BlobStorage, x: T) -> Result<(serde_json::Value, Option<String>, Vec<Blob>)> {
     let pretty = x.to_string();
+
+    let blobs = if let Some(blob) = x.blob() {
+        let blob = bs.load(blob)?;
+        vec![blob]
+    } else {
+        Vec::new()
+    };
+
     let arg = serde_json::to_value(x)?;
-    Ok((arg, Some(pretty)))
+    Ok((arg, Some(pretty), blobs))
 }
 
-fn prepare_args<T: Scopable + Serialize + Model>(db: &Database, filter: &Filter) -> Result<Vec<(serde_json::Value, Option<String>)>> {
-    db.filter::<T>(filter)?
+fn prepare_args<T: Scopable + Serialize + Model>(rl: &Readline, filter: &Filter, param: Option<&String>) -> Result<Vec<(serde_json::Value, Option<String>, Vec<Blob>)>> {
+    let db = rl.db();
+    let bs = rl.blobs();
+    db.filter_with_param::<T>(filter, param)?
         .into_iter()
-        .map(prepare_arg)
+        .map(|x| prepare_arg(bs, x))
         .collect()
 }
 
@@ -113,12 +123,17 @@ pub fn execute(rl: &mut Readline, params: Params, options: HashMap<String, Strin
     let filter = rl.scoped_targets();
 
     let args = match module.source() {
-        Some(Source::Domains) => prepare_args::<Domain>(rl.db(), &filter),
-        Some(Source::Subdomains) => prepare_args::<Subdomain>(rl.db(), &filter),
-        Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl.db(), &filter),
-        Some(Source::Urls) => prepare_args::<Url>(rl.db(), &filter),
-        Some(Source::Emails) => prepare_args::<Email>(rl.db(), &filter),
-        Some(Source::PhoneNumbers) => prepare_args::<PhoneNumber>(rl.db(), &filter),
+        Some(Source::Domains) => prepare_args::<Domain>(rl, &filter, None),
+        Some(Source::Subdomains) => prepare_args::<Subdomain>(rl, &filter, None),
+        Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl, &filter, None),
+        Some(Source::Urls) => prepare_args::<Url>(rl, &filter, None),
+        Some(Source::Emails) => prepare_args::<Email>(rl, &filter, None),
+        Some(Source::PhoneNumbers) => prepare_args::<PhoneNumber>(rl, &filter, None),
+        Some(Source::Networks) => prepare_args::<Network>(rl, &filter, None),
+        Some(Source::Devices) => prepare_args::<Device>(rl, &filter, None),
+        Some(Source::Accounts(service)) => prepare_args::<Account>(rl, &filter, service.as_ref()),
+        Some(Source::Breaches) => prepare_args::<Breach>(rl, &filter, None),
+        Some(Source::Images) => prepare_args::<Image>(rl, &filter, None),
         Some(Source::KeyRing(namespace)) => {
             let keyring = rl.keyring();
             if keyring.is_access_granted(&module, &namespace) {
@@ -126,14 +141,14 @@ pub fn execute(rl: &mut Readline, params: Params, options: HashMap<String, Strin
                     .map(|key| {
                         let pretty = format!("{}:{}", key.namespace, key.access_key);
                         let arg = serde_json::to_value(key)?;
-                        Ok((arg, Some(pretty)))
+                        Ok((arg, Some(pretty), vec![]))
                     })
                     .collect::<Result<Vec<_>>>()
             } else {
                 Ok(vec![])
             }
         },
-        None => Ok(vec![(serde_json::Value::Null, None)]),
+        None => Ok(vec![(serde_json::Value::Null, None, vec![])]),
     }?;
 
     rl.signal_register().catch_ctrl();

src/cmd/scope_cmd.rs

@@ -1,6 +1,6 @@
 use crate::errors::*;
 
-use crate::db;
+use crate::filters::{Target, Filter};
 use crate::shell::Readline;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
@@ -11,46 +11,31 @@ use crate::term;
 #[derive(Debug, StructOpt)]
 #[structopt(author = "",
             raw(global_settings = "&[AppSettings::ColoredHelp]"))]
-pub enum Args {
-    #[structopt(name="domains")]
-    Domains(Filter),
-    #[structopt(name="subdomains")]
-    Subdomains(Filter),
-    #[structopt(name="ipaddrs")]
-    IpAddrs(Filter),
-    #[structopt(name="urls")]
-    Urls(Filter),
-    #[structopt(name="emails")]
-    Emails(Filter),
-    #[structopt(name="phonenumbers")]
-    PhoneNumbers(Filter),
-}
-
-#[derive(Debug, StructOpt)]
-pub struct Filter {
-    args: Vec<String>,
-}
-
-impl Filter {
-    pub fn parse(&self) -> Result<db::Filter> {
-        db::Filter::parse(&self.args)
-    }
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Target,
 }
 
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
-    let rows = match args {
-        Args::Domains(filter) => scope::<Domain>(rl, &filter),
-        Args::Subdomains(filter) => scope::<Subdomain>(rl, &filter),
-        Args::IpAddrs(filter) => scope::<IpAddr>(rl, &filter),
-        Args::Urls(filter) => scope::<Url>(rl, &filter),
-        Args::Emails(filter) => scope::<Email>(rl, &filter),
-        Args::PhoneNumbers(filter) => scope::<PhoneNumber>(rl, &filter),
+    let rows = match args.subcommand {
+        Target::Domains(filter) => scope::<Domain>(rl, &filter),
+        Target::Subdomains(filter) => scope::<Subdomain>(rl, &filter),
+        Target::IpAddrs(filter) => scope::<IpAddr>(rl, &filter),
+        Target::Urls(filter) => scope::<Url>(rl, &filter),
+        Target::Emails(filter) => scope::<Email>(rl, &filter),
+        Target::PhoneNumbers(filter) => scope::<PhoneNumber>(rl, &filter),
+        Target::Devices(filter) => scope::<Device>(rl, &filter),
+        Target::Networks(filter) => scope::<Network>(rl, &filter),
+        Target::Accounts(filter) => scope::<Account>(rl, &filter),
+        Target::Breaches(filter) => scope::<Breach>(rl, &filter),
+        Target::Images(filter) => scope::<Image>(rl, &filter),
     }?;
     term::info(&format!("Updated {} rows", rows));
     Ok(())
 }
 
+#[inline]
 fn scope<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<usize> {
-    rl.db().scope::<T>(&filter.parse()?)
+    T::scope(rl.db(), &filter.parse()?)
 }

src/cmd/select_cmd.rs

@@ -1,8 +1,8 @@
 use crate::errors::*;
 
 use crate::cmd::Cmd;
-use crate::db;
 use crate::db::ttl;
+use crate::filters::{Target, Filter};
 use crate::shell::Readline;
 use serde::Serialize;
 use serde_json;
@@ -17,70 +17,61 @@ use crate::models::*;
 pub struct Args {
     #[structopt(subcommand)]
     subcommand: Target,
-    #[structopt(long="json")]
-    /// Json output
+    /// Print json output
+    #[structopt(long="json", group="output")]
     json: bool,
+    /// Print paths to blobs
+    #[structopt(long="paths", group="output")]
+    paths: bool,
 }
 
-#[derive(Debug, StructOpt)]
-pub enum Target {
-    #[structopt(name="domains")]
-    /// Select domains
-    Domains(Filter),
-    #[structopt(name="subdomains")]
-    /// Select subdomains
-    Subdomains(Filter),
-    #[structopt(name="ipaddrs")]
-    /// Select ipaddrs
-    IpAddrs(Filter),
-    #[structopt(name="urls")]
-    /// Select urls
-    Urls(Filter),
-    #[structopt(name="emails")]
-    /// Select emails
-    Emails(Filter),
-    #[structopt(name="phonenumbers")]
-    /// Select phone numbers
-    PhoneNumbers(Filter),
-    #[structopt(name="devices")]
-    /// Select devices
-    Devices(Filter),
-    #[structopt(name="networks")]
-    /// Select networks
-    Networks(Filter),
+enum Output {
+    Normal,
+    Json,
+    Paths,
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Filter {
-    args: Vec<String>,
+struct Printer<'a, 'b> {
+    rl: &'a mut Readline<'b>,
+    output: Output,
 }
 
-impl Filter {
-    pub fn parse(&self) -> Result<db::Filter> {
-        db::Filter::parse_optional(&self.args)
-    }
-}
+impl<'a, 'b> Printer<'a, 'b> {
+    pub fn new(rl: &'a mut Readline<'b>, args: &Args) -> Printer<'a, 'b> {
+        let output = if args.json {
+            Output::Json
+        } else if args.paths {
+            Output::Paths
+        } else {
+            Output::Normal
+        };
 
-pub struct Printer<'a> {
-    rl: &'a mut Readline,
-    json: bool,
-}
-
-impl<'a> Printer<'a> {
-    pub fn new(rl: &'a mut Readline, json: bool) -> Printer<'a> {
         Printer {
             rl,
-            json,
+            output,
         }
     }
 
     pub fn select<T: Model + Detailed + Serialize>(&self, filter: &Filter) -> Result<()> {
-        for obj in self.rl.db().filter::<T>(&filter.parse()?)? {
-            if self.json {
-                let v = serde_json::to_string(&obj)?;
-                println!("{}", v);
-            } else {
-                println!("{}", obj.detailed(self.rl.db())?);
+        for obj in self.rl.db().filter::<T>(&filter.parse_optional()?)? {
+            match self.output {
+                Output::Normal => println!("{}", obj.detailed(self.rl.db())?),
+                Output::Json => {
+                    let v = serde_json::to_string(&obj)?;
+                    println!("{}", v);
+                },
+                Output::Paths => {
+                    let blob = obj.blob()
+                        .ok_or_else(|| format_err!("This model isn't linked to blob storage"))?;
+
+                    let path = self.rl.blobs()
+                        .join(blob)?;
+
+                    let path = path.to_str()
+                        .ok_or_else(|| format_err!("Path is invalid utf-8"))?;
+
+                    println!("{}", path);
+                },
             }
         }
 
@@ -89,8 +80,8 @@ impl<'a> Printer<'a> {
 }
 
 impl Cmd for Args {
-    fn run(&self, rl: &mut Readline) -> Result<()> {
-        let printer = Printer::new(rl, self.json);
+    fn run(self, rl: &mut Readline) -> Result<()> {
+        let printer = Printer::new(rl, &self);
 
         match &self.subcommand {
             Target::Domains(filter) => printer.select::<Domain>(&filter),
@@ -101,6 +92,9 @@ impl Cmd for Args {
             Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(&filter),
             Target::Devices(filter) => printer.select::<Device>(&filter),
             Target::Networks(filter) => printer.select::<Network>(&filter),
+            Target::Accounts(filter) => printer.select::<Account>(&filter),
+            Target::Breaches(filter) => printer.select::<Breach>(&filter),
+            Target::Images(filter) => printer.select::<Image>(&filter),
         }
     }
 }

src/cmd/target_cmd.rs

@@ -31,12 +31,17 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
 
     if args.filter.is_empty() {
         match source {
-            Source::Domains => select::<Domain>(rl)?,
-            Source::Subdomains => select::<Subdomain>(rl)?,
-            Source::IpAddrs => select::<IpAddr>(rl)?,
-            Source::Urls => select::<Url>(rl)?,
-            Source::Emails => select::<Email>(rl)?,
-            Source::PhoneNumbers => select::<PhoneNumber>(rl)?,
+            Source::Domains => select::<Domain>(rl, None)?,
+            Source::Subdomains => select::<Subdomain>(rl, None)?,
+            Source::IpAddrs => select::<IpAddr>(rl, None)?,
+            Source::Urls => select::<Url>(rl, None)?,
+            Source::Emails => select::<Email>(rl, None)?,
+            Source::PhoneNumbers => select::<PhoneNumber>(rl, None)?,
+            Source::Networks => select::<Network>(rl, None)?,
+            Source::Devices => select::<Device>(rl, None)?,
+            Source::Accounts(service) => select::<Account>(rl, service.as_ref())?,
+            Source::Breaches => select::<Breach>(rl, None)?,
+            Source::Images => select::<Image>(rl, None)?,
             Source::KeyRing(namespace) => {
                 for key in rl.keyring().list_for(&namespace) {
                     println!("{}:{}", key.namespace, key.name);
@@ -64,15 +69,20 @@ fn count_selected(rl: &mut Readline, source: &Source) -> Result<usize> {
         Source::Urls => db.filter::<Url>(&filter)?.len(),
         Source::Emails => db.filter::<Email>(&filter)?.len(),
         Source::PhoneNumbers => db.filter::<PhoneNumber>(&filter)?.len(),
+        Source::Networks => db.filter::<Network>(&filter)?.len(),
+        Source::Devices => db.filter::<Device>(&filter)?.len(),
+        Source::Accounts(service) => db.filter_with_param::<Account>(&filter, service.as_ref())?.len(),
+        Source::Breaches => db.filter::<Breach>(&filter)?.len(),
+        Source::Images => db.filter::<Image>(&filter)?.len(),
         Source::KeyRing(namespace) => rl.keyring().list_for(&namespace).len(),
     };
     Ok(num)
 }
 
-fn select<T: Model + Detailed>(rl: &mut Readline) -> Result<()> {
+fn select<T: Model + Detailed>(rl: &mut Readline, param: Option<&String>) -> Result<()> {
     let filter = rl.scoped_targets();
 
-    for obj in rl.db().filter::<T>(&filter)? {
+    for obj in rl.db().filter_with_param::<T>(&filter, param)? {
         println!("{}", obj.detailed(rl.db())?);
     }
 

src/cmd/workspace_cmd.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 
+use crate::blobs::BlobStorage;
 use crate::db::Database;
 use crate::shell::Readline;
 use structopt::StructOpt;
@@ -18,7 +19,9 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
 
     if let Some(workspace) = args.workspace {
+        let blobs = BlobStorage::workspace(&workspace)?;
         let db = Database::establish(workspace)?;
+        rl.set_blobstorage(blobs);
         rl.set_db(db);
     } else {
         for x in workspaces::list()? {

src/complete.rs

@@ -33,7 +33,10 @@ impl CmdCompleter {
                             "emails",
                             "phonenumbers",
                             "devices",
-                            "networks"];
+                            "networks",
+                            "accounts",
+                            "breaches",
+                            "images"];
 
             let results: Vec<String> = options.iter()
                 .filter(|x| x.starts_with(arg))
@@ -93,7 +96,10 @@ impl Completer for CmdCompleter {
                                         "email",
                                         "phonenumber",
                                         "device",
-                                        "network"];
+                                        "network",
+                                        "account",
+                                        "breach",
+                                        "image"];
 
                         let results: Vec<String> = options.iter()
                             .filter(|x| x.starts_with(arg))

src/config.rs

@@ -1,5 +1,6 @@
 use dirs;
 use crate::errors::*;
+use std::collections::HashMap;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::net::SocketAddr;
@@ -11,6 +12,8 @@ pub struct Config {
     #[serde(default)]
     pub core: CoreConfig,
     #[serde(default)]
+    pub namespaces: HashMap<String, PathBuf>,
+    #[serde(default)]
     pub network: NetworkConfig,
 }
 
@@ -50,12 +53,15 @@ impl Config {
 pub struct CoreConfig {
     #[serde(default="default_registry")]
     pub registry: String,
+    #[serde(default, rename="no-autoupdate")]
+    pub no_autoupdate: bool,
 }
 
 impl Default for CoreConfig {
     fn default() -> CoreConfig {
         CoreConfig {
             registry: default_registry(),
+            no_autoupdate: false,
         }
     }
 }

src/db/mod.rs

@@ -44,6 +44,10 @@ pub enum Family {
     Device,
     Network,
     NetworkDevice,
+    Account,
+    Breach,
+    BreachEmail,
+    Image,
 }
 
 impl FromStr for Family {
@@ -61,6 +65,10 @@ impl FromStr for Family {
             "device" => Family::Device,
             "network" => Family::Network,
             "network-device" => Family::NetworkDevice,
+            "account" => Family::Account,
+            "breach" => Family::Breach,
+            "breach-email" => Family::BreachEmail,
+            "image" => Family::Image,
             _ => bail!("Unknown object family"),
         })
     }
@@ -109,80 +117,22 @@ impl Database {
     }
 
     /// Returns true if we didn't have this value yet
-    pub fn insert_generic(&self, object: &Insert) -> Result<Option<(DbChange, i32)>> {
+    pub fn insert_generic(&self, object: Insert) -> Result<Option<(DbChange, i32)>> {
         match object {
-            Insert::Domain(object) => self.insert_struct(NewDomain {
-                value: &object.value,
-            }),
-            Insert::Subdomain(object) => self.insert_struct(NewSubdomain {
-                domain_id: object.domain_id,
-                value: &object.value,
-                resolvable: object.resolvable,
-            }),
-            Insert::IpAddr(object) => self.insert_struct(NewIpAddr {
-                family: &object.family,
-                value: &object.value,
-                continent: object.continent.as_ref(),
-                continent_code: object.continent_code.as_ref(),
-                country: object.country.as_ref(),
-                country_code: object.country_code.as_ref(),
-                city: object.city.as_ref(),
-                longitude: object.longitude,
-                latitude: object.latitude,
-                asn: object.asn,
-                as_org: object.as_org.as_ref(),
-                description: object.description.as_ref(),
-                reverse_dns: object.reverse_dns.as_ref(),
-            }),
-            Insert::SubdomainIpAddr(object) => self.insert_subdomain_ipaddr_struct(&NewSubdomainIpAddr {
-                subdomain_id: object.subdomain_id,
-                ip_addr_id: object.ip_addr_id,
-            }),
-            Insert::Url(object) => self.insert_struct(NewUrl {
-                subdomain_id: object.subdomain_id,
-                value: &object.value,
-                path: &object.path,
-                status: object.status,
-                body: object.body.as_ref(),
-                online: object.online,
-                title: object.title.as_ref(),
-                redirect: object.redirect.as_ref(),
-            }),
-            Insert::Email(object) => self.insert_struct(NewEmail {
-                value: &object.value,
-                valid: object.valid,
-            }),
-            Insert::PhoneNumber(object) => self.insert_struct(NewPhoneNumber {
-                value: &object.value,
-                name: object.name.as_ref(),
-                valid: object.valid,
-                last_online: object.last_online,
-                country: object.country.as_ref(),
-                carrier: object.carrier.as_ref(),
-                line: object.line.as_ref(),
-                is_ported: object.is_ported,
-                last_ported: object.last_ported,
-                caller_name: object.caller_name.as_ref(),
-                caller_type: object.caller_type.as_ref(),
-            }),
-            Insert::Device(object) => self.insert_struct(NewDevice {
-                value: &object.value,
-                name: object.name.as_ref(),
-                hostname: object.hostname.as_ref(),
-                vendor: object.vendor.as_ref(),
-                last_seen: object.last_seen,
-            }),
-            Insert::Network(object) => self.insert_struct(NewNetwork {
-                value: &object.value,
-                latitude: object.latitude,
-                longitude: object.longitude,
-            }),
-            Insert::NetworkDevice(object) => self.insert_network_device_struct(&NewNetworkDevice {
-                network_id: object.network_id,
-                device_id: object.device_id,
-                ipaddr: object.ipaddr.as_ref(),
-                last_seen: object.last_seen,
-            }),
+            Insert::Domain(object) => self.insert_struct(object),
+            Insert::Subdomain(object) => self.insert_struct(object),
+            Insert::IpAddr(object) => self.insert_struct(object),
+            Insert::SubdomainIpAddr(object) => self.insert_subdomain_ipaddr_struct(&object),
+            Insert::Url(object) => self.insert_struct(object),
+            Insert::Email(object) => self.insert_struct(object),
+            Insert::PhoneNumber(object) => self.insert_struct(object),
+            Insert::Device(object) => self.insert_struct(object),
+            Insert::Network(object) => self.insert_struct(object),
+            Insert::NetworkDevice(object) => self.insert_network_device_struct(&object),
+            Insert::Account(object) => self.insert_struct(object),
+            Insert::Breach(object) => self.insert_struct(object),
+            Insert::BreachEmail(object) => self.insert_breach_email_struct(object),
+            Insert::Image(object) => self.insert_struct(object),
         }
     }
 
@@ -221,8 +171,8 @@ impl Database {
     }
 
     pub fn insert_network_device_struct(&self, network_device: &NewNetworkDevice) -> Result<Option<(DbChange, i32)>> {
-        if let Some(subdomain_ipaddr_id) = NetworkDevice::get_id_opt(self, &(network_device.network_id, network_device.device_id))? {
-            Ok(Some((DbChange::None, subdomain_ipaddr_id)))
+        if let Some(network_device_id) = NetworkDevice::get_id_opt(self, &(network_device.network_id, network_device.device_id))? {
+            Ok(Some((DbChange::None, network_device_id)))
         } else {
             diesel::insert_into(network_devices::table)
                 .values(network_device)
@@ -232,6 +182,28 @@ impl Database {
         }
     }
 
+    pub fn insert_breach_email_struct(&self, obj: NewBreachEmail) -> Result<Option<(DbChange, i32)>> {
+        let value = &(obj.breach_id, obj.email_id, obj.password.clone());
+
+        if let Some(existing) = BreachEmail::get_opt(self, value)? {
+            let id = <BreachEmail as Model>::id(&existing);
+
+            let update = obj.upsert(&existing);
+            if update.is_dirty() {
+                update.apply(&self)?;
+                Ok(Some((DbChange::Update(update.generic()), id)))
+            } else {
+                Ok(Some((DbChange::None, id)))
+            }
+        } else {
+            diesel::insert_into(breach_emails::table)
+                .values(obj)
+                .execute(&self.db)?;
+            let id = BreachEmail::get_id(self, value)?;
+            Ok(Some((DbChange::Insert, id)))
+        }
+    }
+
     //
 
     pub fn update_generic(&self, object: &Update) -> Result<i32> {
@@ -244,6 +216,9 @@ impl Database {
             Update::Device(object) => self.update_device(object),
             Update::Network(object) => self.update_network(object),
             Update::NetworkDevice(object) => self.update_network_device(object),
+            Update::Account(object) => self.update_account(object),
+            Update::BreachEmail(object) => self.update_breach_email(object),
+            Update::Image(object) => self.update_image(object),
         }
     }
 
@@ -311,6 +286,30 @@ impl Database {
         Ok(network_device.id)
     }
 
+    pub fn update_account(&self, account: &AccountUpdate) -> Result<i32> {
+        use crate::schema::accounts::columns::*;
+        diesel::update(accounts::table.filter(id.eq(account.id)))
+            .set(account)
+            .execute(&self.db)?;
+        Ok(account.id)
+    }
+
+    pub fn update_breach_email(&self, breach_email: &BreachEmailUpdate) -> Result<i32> {
+        use crate::schema::breach_emails::columns::*;
+        diesel::update(breach_emails::table.filter(id.eq(breach_email.id)))
+            .set(breach_email)
+            .execute(&self.db)?;
+        Ok(breach_email.id)
+    }
+
+    pub fn update_image(&self, image: &ImageUpdate) -> Result<i32> {
+        use crate::schema::images::columns::*;
+        diesel::update(images::table.filter(id.eq(image.id)))
+            .set(image)
+            .execute(&self.db)?;
+        Ok(image.id)
+    }
+
     fn get_opt_typed<T: Model + Scopable>(&self, value: &T::ID) -> Result<Option<i32>> {
         match T::get_opt(self, &value)? {
             Some(ref obj) if obj.scoped() => Ok(Some(obj.id())),
@@ -330,6 +329,10 @@ impl Database {
             Family::Device => self.get_opt_typed::<Device>(&value),
             Family::Network => self.get_opt_typed::<Network>(&value),
             Family::NetworkDevice => bail!("Unsupported operation"),
+            Family::Account => self.get_opt_typed::<Account>(&value),
+            Family::Breach => self.get_opt_typed::<Breach>(&value),
+            Family::BreachEmail => bail!("Unsupported operation"),
+            Family::Image => self.get_opt_typed::<Image>(&value),
         }
     }
 
@@ -343,16 +346,11 @@ impl Database {
         T::filter(self, filter)
     }
 
-    pub fn scope<T: Scopable>(&self, filter: &Filter) -> Result<usize> {
-        T::scope(self, filter)
-    }
-
-    pub fn noscope<T: Scopable>(&self, filter: &Filter) -> Result<usize> {
-        T::noscope(self, filter)
-    }
-
-    pub fn delete<T: Model>(&self, filter: &Filter) -> Result<usize> {
-        T::delete(self, filter)
+    pub fn filter_with_param<T: Model>(&self, filter: &Filter, param: Option<&String>) -> Result<Vec<T>> {
+        match param {
+            Some(param) => T::filter_with_param(self, filter, param),
+            _ => T::filter(self, filter),
+        }
     }
 }
 
@@ -398,6 +396,12 @@ impl Filter {
         let mut expect_value = false;
 
         for arg in args {
+            if ["=", "!=", "<", ">", "<=", ">=", "like"].contains(&arg.to_lowercase().as_str()) {
+                expect_value = true;
+                query += &format!(" {}", arg);
+                continue;
+            }
+
             if let Some(idx) = arg.find('=') {
                 if idx != 0 {
                     let (key, value) = arg.split_at(idx);
@@ -411,10 +415,6 @@ impl Filter {
                 query.push_str(&Self::escape(arg));
                 expect_value = false;
             } else {
-                if ["=", "!=", "like"].contains(&arg.to_lowercase().as_str()) {
-                    expect_value = true;
-                }
-
                 query += &format!(" {}", arg);
             }
         }
@@ -538,4 +538,44 @@ mod tests {
                                     ]).unwrap();
         assert_eq!(filter, Filter::new(" value = 'a''b'"));
     }
+
+    #[test]
+    fn test_filter_greater() {
+        let filter = Filter::parse(&["where".to_string(),
+                                     "value".to_string(),
+                                     ">".to_string(),
+                                     "123".to_string(),
+                                    ]).unwrap();
+        assert_eq!(filter, Filter::new(" value > '123'"));
+    }
+
+    #[test]
+    fn test_filter_smaller() {
+        let filter = Filter::parse(&["where".to_string(),
+                                     "value".to_string(),
+                                     "<".to_string(),
+                                     "123".to_string(),
+                                    ]).unwrap();
+        assert_eq!(filter, Filter::new(" value < '123'"));
+    }
+
+    #[test]
+    fn test_filter_greater_equal() {
+        let filter = Filter::parse(&["where".to_string(),
+                                     "value".to_string(),
+                                     ">=".to_string(),
+                                     "123".to_string(),
+                                    ]).unwrap();
+        assert_eq!(filter, Filter::new(" value >= '123'"));
+    }
+
+    #[test]
+    fn test_filter_smaller_equal() {
+        let filter = Filter::parse(&["where".to_string(),
+                                     "value".to_string(),
+                                     "<=".to_string(),
+                                     "123".to_string(),
+                                    ]).unwrap();
+        assert_eq!(filter, Filter::new(" value <= '123'"));
+    }
 }

src/engine/ctx.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 
+use crate::blobs::Blob;
 use crate::db::Family;
 use crate::engine::{Environment, Reporter};
 use crate::geoip::{GeoIP, AsnDB};
@@ -120,6 +121,18 @@ pub trait State {
     fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest;
 
     fn register_in_jar(&self, session: &str, key: String, value: String);
+
+    fn register_blob(&self, blob: Blob) -> String;
+
+    fn get_blob(&self, id: &str) -> Result<Arc<Blob>>;
+
+    fn persist_blob(&self, id: &str) -> Result<()> {
+        let blob = self.get_blob(id)?;
+        self.send(&Event::Blob(blob.as_ref().clone()));
+        let reply = self.recv()?;
+        let reply: result::Result<(), String> = serde_json::from_value(reply)?;
+        reply.map_err(|err| format_err!("Failed to store blob: {:?}", err))
+    }
 }
 
 #[derive(Debug)]
@@ -127,8 +140,10 @@ pub struct LuaState {
     error: Mutex<Option<Error>>,
     logger: Arc<Mutex<Box<Reporter>>>,
     socket_sessions: Mutex<HashMap<String, Arc<Mutex<Socket>>>>,
+    blobs: Mutex<HashMap<String, Arc<Blob>>>,
     http_sessions: Mutex<HashMap<String, HttpSession>>,
     http: chrootable_https::Client<Resolver>,
+
     verbose: u64,
     keyring: Vec<KeyRingEntry>, // TODO: maybe hashmap
     dns_config: Resolver,
@@ -171,9 +186,9 @@ impl State for LuaState {
         self.verbose
     }
 
-    fn keyring(&self, namespace: &str) -> Vec<&KeyRingEntry> {
+    fn keyring(&self, query: &str) -> Vec<&KeyRingEntry> {
         self.keyring.iter()
-            .filter(|x| x.namespace == namespace)
+            .filter(|x| x.matches(query))
             .collect()
     }
 
@@ -245,6 +260,23 @@ impl State for LuaState {
             session.cookies.register_in_jar(key, value);
         }
     }
+
+    fn register_blob(&self, blob: Blob) -> String {
+        let id = blob.id.clone();
+
+        let mut mtx = self.blobs.lock().unwrap();
+        mtx.insert(id.clone(), Arc::new(blob));
+        debug!("Registered blob: {:?}", id);
+
+        id
+    }
+
+    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
+        let mtx = self.blobs.lock().unwrap();
+        let blob = mtx.get(id)
+            .ok_or_else(|| format_err!("Invalid blob reference"))?;
+        Ok(blob.clone())
+    }
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -269,6 +301,7 @@ fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<Reporter>>>) -> (hlua::Lua<'a
         error: Mutex::new(None),
         logger,
         socket_sessions: Mutex::new(HashMap::new()),
+        blobs: Mutex::new(HashMap::new()),
         http_sessions: Mutex::new(HashMap::new()),
         http,
 
@@ -282,7 +315,13 @@ fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<Reporter>>>) -> (hlua::Lua<'a
         options: env.options,
     });
 
+    debug!("Adding all blobs from StartCommand");
+    for blob in env.blobs {
+        state.register_blob(blob);
+    }
+
     runtime::clear_err(&mut lua, state.clone());
+    runtime::create_blob(&mut lua, state.clone());
     runtime::datetime(&mut lua, state.clone());
     runtime::db_add(&mut lua, state.clone());
     runtime::db_add_ttl(&mut lua, state.clone());
@@ -300,19 +339,28 @@ fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<Reporter>>>) -> (hlua::Lua<'a
     runtime::http_mksession(&mut lua, state.clone());
     runtime::http_request(&mut lua, state.clone());
     runtime::http_send(&mut lua, state.clone());
+    runtime::img_exif(&mut lua, state.clone());
+    runtime::img_load(&mut lua, state.clone());
+    runtime::img_nudity(&mut lua, state.clone());
     runtime::info(&mut lua, state.clone());
     runtime::json_decode(&mut lua, state.clone());
     runtime::json_decode_stream(&mut lua, state.clone());
     runtime::json_encode(&mut lua, state.clone());
     runtime::keyring(&mut lua, state.clone());
     runtime::last_err(&mut lua, state.clone());
+    runtime::md5(&mut lua, state.clone());
     runtime::pgp_pubkey(&mut lua, state.clone());
     runtime::pgp_pubkey_armored(&mut lua, state.clone());
     runtime::print(&mut lua, state.clone());
     runtime::psl_domain_from_dns_name(&mut lua, state.clone());
     runtime::regex_find(&mut lua, state.clone());
     runtime::regex_find_all(&mut lua, state.clone());
+    runtime::semver_match(&mut lua, state.clone());
+    runtime::sha1(&mut lua, state.clone());
+    runtime::sha2_256(&mut lua, state.clone());
+    runtime::sha2_512(&mut lua, state.clone());
     runtime::sleep(&mut lua, state.clone());
+    runtime::sn0int_version(&mut lua, state.clone());
     runtime::sock_connect(&mut lua, state.clone());
     runtime::sock_send(&mut lua, state.clone());
     runtime::sock_recv(&mut lua, state.clone());
@@ -412,6 +460,7 @@ com
             dns_config,
             proxy,
             options: HashMap::new(),
+            blobs: Vec::new(),
             psl,
             geoip,
             asn,

src/engine/isolation.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
 use crate::engine::{Environment, Module, Reporter};
 use crate::geoip::{GeoIP, AsnDB, Maxmind};
 use crate::keyring::KeyRingEntry;
@@ -25,6 +26,7 @@ pub struct StartCommand {
     options: HashMap<String, String>,
     module: Module,
     arg: serde_json::Value,
+    blobs: Vec<Blob>,
 }
 
 impl StartCommand {
@@ -35,6 +37,7 @@ impl StartCommand {
                options: HashMap<String, String>,
                module: Module,
                arg: serde_json::Value,
+               blobs: Vec<Blob>,
     ) -> StartCommand {
         StartCommand {
             verbose,
@@ -44,6 +47,7 @@ impl StartCommand {
             options,
             module,
             arg,
+            blobs,
         }
     }
 }
@@ -181,6 +185,7 @@ pub fn spawn_module(module: Module,
                     has_stdin: bool,
                     proxy: Option<SocketAddr>,
                     options: HashMap<String, String>,
+                    blobs: Vec<Blob>,
 ) -> Result<ExitEvent> {
     let dns_config = Resolver::from_system()?;
 
@@ -191,13 +196,14 @@ pub fn spawn_module(module: Module,
     };
 
     let mut supervisor = Supervisor::setup(&module)?;
-    supervisor.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg))?;
+    supervisor.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
 
     let exit = loop {
         match supervisor.recv()? {
             Event::Log(event) => tx.send(Event2::Log(event)),
             Event::Database(object) => supervisor.send_event_callback(object, &tx),
             Event::Stdio(object) => object.apply(&mut supervisor, tx, &mut reader),
+            Event::Blob(blob) => supervisor.send_event_callback(blob, &tx),
             Event::Exit(event) => {
                 if let ExitEvent::Err(err) = &event {
                     tx.send(Event2::Log(LogEvent::Error(err.clone())));
@@ -227,6 +233,7 @@ pub fn run_worker(geoip: Vec<u8>, asn: Vec<u8>, psl: &str) -> Result<()> {
         dns_config: start.dns_config,
         proxy: start.proxy,
         options: start.options,
+        blobs: start.blobs,
         psl,
         geoip,
         asn,

src/engine/mod.rs

@@ -1,5 +1,7 @@
 use crate::errors::*;
 
+use crate::blobs::Blob;
+use crate::config::Config;
 use crate::geoip::{GeoIP, AsnDB};
 use crate::json::LuaJsonValue;
 use crate::keyring::KeyRingEntry;
@@ -17,6 +19,7 @@ use chrootable_https::dns::Resolver;
 use crate::psl::Psl;
 use crate::paths;
 use std::cmp::Ordering;
+use std::path::Path;
 use crate::term;
 use crate::worker::{self, Event};
 
@@ -33,24 +36,27 @@ pub struct Environment {
     pub dns_config: Resolver,
     pub proxy: Option<SocketAddr>,
     pub options: HashMap<String, String>,
+    pub blobs: Vec<Blob>,
     pub psl: Psl,
     pub geoip: GeoIP,
     pub asn: AsnDB,
 }
 
 #[derive(Debug)]
-pub struct Engine {
+pub struct Engine<'a> {
     path: PathBuf,
     modules: HashMap<String, Vec<Module>>,
+    config: &'a Config
 }
 
-impl Engine {
-    pub fn new(verbose_init: bool) -> Result<Engine> {
+impl<'a> Engine<'a> {
+    pub fn new(verbose_init: bool, config: &'a Config) -> Result<Engine> {
         let path = paths::module_dir()?;
 
         let mut engine = Engine {
             path,
             modules: HashMap::new(),
+            config,
         };
 
         if verbose_init {
@@ -72,53 +78,103 @@ impl Engine {
         Ok(modules)
     }
 
+    pub fn private_modules(path: &Path) -> Result<bool> {
+        let metadata = fs::symlink_metadata(&path)?.file_type();
+        if metadata.is_symlink() {
+            debug!("Folder is a symlink, flagging modules as private");
+            return Ok(true);
+        }
+
+        if path.join(".git").exists() {
+            debug!("Folder is a git repo, flagging modules as private");
+            return Ok(true);
+        }
+
+        Ok(false)
+    }
+
     pub fn reload_modules_quiet(&mut self) -> Result<()> {
         self.modules = HashMap::new();
 
         for author in fs::read_dir(&self.path)? {
             let author = author?;
+            let path = author.path();
 
-            if !author.path().is_dir() {
+            if !path.is_dir() {
                 continue;
             }
 
+            let private_modules = Self::private_modules(&path)?;
+
             let author_name = author.file_name()
                                     .into_string()
                                     .map_err(|_| format_err!("Failed to decode filename"))?;
-            for module in fs::read_dir(&author.path())? {
-                let module = module?;
-                let module_name = module.file_name()
-                                        .into_string()
-                                        .map_err(|_| format_err!("Failed to decode filename"))?;
 
-                // find last instance of .lua in filename, if any
-                let (module_name, ext) = if let Some(idx) = module_name.rfind(".lua") {
-                    module_name.split_at(idx)
-                } else {
-                    // TODO: show warning
-                    continue;
-                };
+            self.load_module_folder(&path, &author_name, private_modules)?;
+        }
 
-                // if .lua is not at the end, skip
-                if ext != ".lua" {
-                    // TODO: show warning
-                    continue;
-                }
+        for (author, folder) in &self.config.namespaces {
+            let folder = if folder.is_absolute() {
+                folder.to_owned()
+            } else {
+                let folder = folder.strip_prefix("~/")
+                    .unwrap_or(&folder);
 
-                let path = module.path();
-                let module_name = module_name.to_string();
-                let module = Module::load(&path, &author_name, &module_name)
-                    .context(format!("Failed to parse {}/{}", author_name, module_name))?;
+                dirs::home_dir()
+                    .ok_or_else(|| format_err!("Failed to find home folder"))?
+                    .join(folder)
+            };
 
-                for key in &[&module_name, &format!("{}/{}", author_name, module_name)] {
-                    if !self.modules.contains_key(*key) {
-                        self.modules.insert(key.to_string(), Vec::new());
-                    }
+            self.load_module_folder(&folder, &author, true)?;
+        }
 
-                    let vec = self.modules.get_mut(*key).unwrap();
-                    vec.push(module.clone());
-                }
+        Ok(())
+    }
+
+    pub fn load_module_folder(&mut self, folder: &Path, author_name: &str, private_modules: bool) -> Result<()> {
+        debug!("Loading modules from {:?}", folder);
+
+        for module in fs::read_dir(folder)? {
+            let module = module?;
+            let module_name = module.file_name()
+                                    .into_string()
+                                    .map_err(|_| format_err!("Failed to decode filename"))?;
+
+            // find last instance of .lua in filename, if any
+            let (module_name, ext) = if let Some(idx) = module_name.rfind(".lua") {
+                module_name.split_at(idx)
+            } else {
+                // TODO: show warning
+                continue;
+            };
+
+            // if .lua is not at the end, skip
+            if ext != ".lua" {
+                // TODO: show warning
+                continue;
             }
+
+            if let Err(err) = self.load_single_module(&module.path(), &author_name, &module_name, private_modules) {
+                let root = err.find_root_cause();
+                term::warn(&format!("Failed to load {}/{}: {}", author_name, module_name, root));
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn load_single_module(&mut self, path: &Path, author_name: &str, module_name: &str, private_module: bool) -> Result<()> {
+        let module_name = module_name.to_string();
+        let module = Module::load(path, &author_name, &module_name, private_module)
+            .context(format!("Failed to parse {}/{}", author_name, module_name))?;
+
+        for key in &[&module_name, &format!("{}/{}", author_name, module_name)] {
+            if !self.modules.contains_key(*key) {
+                self.modules.insert(key.to_string(), Vec::new());
+            }
+
+            let vec = self.modules.get_mut(*key).unwrap();
+            vec.push(module.clone());
         }
 
         Ok(())
@@ -161,11 +217,12 @@ pub struct Module {
     version: String,
     source: Option<Source>,
     keyring_access: Vec<String>,
+    private_module: bool,
     script: Script,
 }
 
 impl Module {
-    pub fn load(path: &PathBuf, author: &str, name: &str) -> Result<Module> {
+    pub fn load(path: &Path, author: &str, name: &str, private_module: bool) -> Result<Module> {
         debug!("Loading lua module {}/{} from {:?}", author, name, path);
         let code = fs::read_to_string(path)
             .context("Failed to read module")?;
@@ -182,10 +239,12 @@ impl Module {
             version: metadata.version,
             source: metadata.source,
             keyring_access: metadata.keyring_access,
+            private_module,
             script,
         })
     }
 
+    #[inline]
     pub fn name(&self) -> &str {
         &self.name
     }
@@ -194,6 +253,7 @@ impl Module {
         format!("{}/{}", self.author, self.name)
     }
 
+    #[inline]
     pub fn id(&self) -> ModuleID {
         ModuleID {
             author: self.author.to_string(),
@@ -201,27 +261,37 @@ impl Module {
         }
     }
 
+    #[inline]
     pub fn description(&self) -> &str {
         &self.description
     }
 
+    #[inline]
     pub fn version(&self) -> &str {
         &self.version
     }
 
+    #[inline]
     pub fn source(&self) -> &Option<Source> {
         &self.source
     }
 
+    #[inline]
     pub fn keyring_access(&self) -> &[String] {
         &self.keyring_access
     }
 
+    #[inline]
+    pub fn is_private(&self) -> bool {
+        self.private_module
+    }
+
     pub fn run(&self, env: Environment, reporter: Arc<Mutex<Box<Reporter>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Executing lua script {}", self.canonical());
         self.script.run(env, reporter, arg.into())
     }
 
+    #[inline]
     fn cmp_canonical(&self, other: &Module) -> Ordering {
         if self.author == other.author {
             self.name.cmp(&other.name)

src/filters.rs

@@ -0,0 +1,57 @@
+use crate::errors::*;
+
+use crate::db;
+use structopt::StructOpt;
+
+
+#[derive(Debug, StructOpt)]
+pub enum Target {
+    /// On domains
+    #[structopt(name="domains")]
+    Domains(Filter),
+    /// On subdomains
+    #[structopt(name="subdomains")]
+    Subdomains(Filter),
+    /// On ipaddrs
+    #[structopt(name="ipaddrs")]
+    IpAddrs(Filter),
+    /// On urls
+    #[structopt(name="urls")]
+    Urls(Filter),
+    /// On emails
+    #[structopt(name="emails")]
+    Emails(Filter),
+    /// On phone numbers
+    #[structopt(name="phonenumbers")]
+    PhoneNumbers(Filter),
+    /// On devices
+    #[structopt(name="devices")]
+    Devices(Filter),
+    /// On networks
+    #[structopt(name="networks")]
+    Networks(Filter),
+    /// On accounts
+    #[structopt(name="accounts")]
+    Accounts(Filter),
+    /// On breaches
+    #[structopt(name="breaches")]
+    Breaches(Filter),
+    /// On images
+    #[structopt(name="images")]
+    Images(Filter),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Filter {
+    args: Vec<String>,
+}
+
+impl Filter {
+    pub fn parse_optional(&self) -> Result<db::Filter> {
+        db::Filter::parse_optional(&self.args)
+    }
+
+    pub fn parse(&self) -> Result<db::Filter> {
+        db::Filter::parse(&self.args)
+    }
+}

src/geoip/mod.rs

@@ -77,7 +77,8 @@ pub trait Maxmind: Sized {
         debug!("Downloading {:?}...", url);
         let client = Client::with_system_resolver()?;
         let resp = client.get(url)
-            .wait_for_response()?;
+            .wait_for_response()
+            .context("http request failed")?;
         debug!("Downloaded {} bytes", resp.body.len());
         archive::extract(&mut &resp.body[..], filter, path)?;
         Ok(())

src/gfx/exif.rs

@@ -0,0 +1,180 @@
+use crate::errors::*;
+
+use exif;
+use std::io;
+
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct Location {
+    latitude: f64,
+    longitude: f64,
+}
+
+impl Location {
+    pub fn try_from(fields: &[exif::Field]) -> Result<Location> {
+        let mut builder = LocationBuilder::default();
+        fields.iter()
+            .map(|f| builder.add_one(f))
+            .collect::<Result<()>>()?;
+        builder.build()
+    }
+}
+
+#[derive(Debug, Default)]
+struct LocationBuilder {
+    latitude: Option<f64>,
+    latitude_ref: Option<f64>,
+    longitude: Option<f64>,
+    longitude_ref: Option<f64>,
+}
+
+impl LocationBuilder {
+    fn add_one(&mut self, f: &exif::Field) -> Result<()> {
+        debug!("Exif tag: {:?}, {}", f.tag, f.value.display_as(f.tag));
+        match f.tag {
+            exif::Tag::GPSLatitudeRef => self.latitude_ref = Some(cardinal_direction_modifier(&f.value)?),
+            exif::Tag::GPSLongitudeRef => self.longitude_ref = Some(cardinal_direction_modifier(&f.value)?),
+            exif::Tag::GPSLatitude => self.latitude = Some(dms_to_float(&f.value)?),
+            exif::Tag::GPSLongitude => self.longitude = Some(dms_to_float(&f.value)?),
+            _ => (),
+        };
+        Ok(())
+    }
+
+    fn build(self) -> Result<Location> {
+        let latitude = self.latitude
+            .ok_or_else(|| format_err!("Missing latitude field"))?;
+        let latitude_ref = self.latitude_ref
+            .ok_or_else(|| format_err!("Missing latitude field"))?;
+
+        let longitude = self.longitude
+            .ok_or_else(|| format_err!("Missing latitude field"))?;
+        let longitude_ref = self.longitude_ref
+            .ok_or_else(|| format_err!("Missing latitude field"))?;
+
+        Ok(Location {
+            latitude: latitude * latitude_ref,
+            longitude: longitude * longitude_ref,
+        })
+    }
+}
+
+pub fn gps(img: &[u8]) -> Result<Option<Location>> {
+    let mut buf = io::BufReader::new(img);
+    let reader = exif::Reader::new(&mut buf)?;
+    let fields = reader.fields();
+    debug!("Exif fields: {:?}", fields);
+
+    let location = Location::try_from(fields).ok();
+    Ok(location)
+}
+
+pub fn dms_to_float(dms: &exif::Value) -> Result<f64> {
+    let dms = match dms {
+        exif::Value::Rational(dms) => dms,
+        _ => panic!("Unexpected exif value for dms"),
+    };
+
+    if dms.len() != 3 {
+        bail!("Incorrect numbers for floats for dms");
+    }
+
+    let degrees = dms[0].to_f64();
+    let minutes = dms[1].to_f64();
+    let seconds = dms[2].to_f64();
+
+    let float = degrees + minutes/60.0 + seconds/3600.0;
+    let float = (float * 1000000.0).round() / 1000000.0;
+    Ok(float)
+}
+
+pub fn cardinal_direction_modifier(value: &exif::Value) -> Result<f64> {
+    match value {
+        exif::Value::Ascii(s) => {
+            let s = s.get(0)
+                .ok_or_else(|| format_err!("Cardinal direction value is empty"))?;
+
+            match s.get(0) {
+                Some(b'N') => Ok(1.0),
+                Some(b'S') => Ok(-1.0),
+                Some(b'E') => Ok(1.0),
+                Some(b'W') => Ok(-1.0),
+                _ => bail!("Unexpected cardinal direction"),
+            }
+        },
+        _ => bail!("Unexpected exif value"),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_init;
+
+    #[test]
+    fn verify_exif_location() {
+        test_init();
+
+        let location = Location::try_from(&[
+            exif::Field {
+                tag: exif::Tag::GPSLatitudeRef,
+                thumbnail: false,
+                value: exif::Value::Ascii(vec![&[b'N']]),
+            }, exif::Field {
+                tag: exif::Tag::GPSLongitudeRef,
+                thumbnail: false,
+                value: exif::Value::Ascii(vec![&[b'E']]),
+            }, exif::Field {
+                tag: exif::Tag::GPSLatitude,
+                thumbnail: false,
+                value: exif::Value::Rational(vec![exif::Rational {
+                    num: 43,
+                    denom: 1,
+                }, exif::Rational {
+                    num: 28,
+                    denom: 1,
+                }, exif::Rational {
+                    num: 176399999,
+                    denom: 100000000,
+                }]),
+            }, exif::Field {
+                tag: exif::Tag::GPSLongitude,
+                thumbnail: false,
+                value: exif::Value::Rational(vec![exif::Rational {
+                    num: 11,
+                    denom: 1,
+                }, exif::Rational {
+                    num: 53,
+                    denom: 1,
+                }, exif::Rational {
+                    num: 742199999,
+                    denom: 100000000,
+                }]),
+            },
+        ]).unwrap();
+        println!("{:?}", location);
+
+        assert_eq!(location, Location {
+            latitude: 43.467157,
+            longitude: 11.885395
+        });
+    }
+
+    #[test]
+    fn verify_dms() {
+        test_init();
+
+        let latitude = dms_to_float(&exif::Value::Rational(vec![exif::Rational {
+            num: 43,
+            denom: 1,
+        }, exif::Rational {
+            num: 28,
+            denom: 1,
+        }, exif::Rational {
+            num: 176399999,
+            denom: 100000000,
+        }])).unwrap();
+
+        assert_eq!(latitude, 43.467157);
+    }
+}

src/gfx/mod.rs

@@ -0,0 +1,124 @@
+use crate::errors::*;
+use image::{self, DynamicImage, GenericImageView};
+
+pub mod exif;
+
+
+#[derive(Debug)]
+pub enum ImageFormat {
+    PNG,
+    JPEG,
+    GIF,
+    WEBP,
+    TIFF,
+    BMP,
+    ICO,
+}
+
+impl ImageFormat {
+    pub fn mime(&self) -> &str {
+        // https://www.iana.org/assignments/media-types/media-types.xhtml#image
+        // /etc/nginx/mime.types
+        match self {
+            ImageFormat::PNG => "image/png",
+            ImageFormat::JPEG => "image/jpeg",
+            ImageFormat::GIF => "image/gif",
+            ImageFormat::WEBP => "image/webp",
+            ImageFormat::TIFF => "image/tiff",
+            ImageFormat::BMP => "image/bmp",
+            ImageFormat::ICO => "image/vnd.microsoft.icon",
+        }
+    }
+
+    pub fn try_from(format: &image::ImageFormat) -> Result<ImageFormat> {
+        use image::ImageFormat::*;
+        match format {
+            PNG => Ok(ImageFormat::PNG),
+            JPEG => Ok(ImageFormat::JPEG),
+            GIF => Ok(ImageFormat::GIF),
+            WEBP => Ok(ImageFormat::WEBP),
+            TIFF => Ok(ImageFormat::TIFF),
+            BMP => Ok(ImageFormat::BMP),
+            ICO => Ok(ImageFormat::ICO),
+
+            HDR | PNM | TGA => bail!("Unsupported format: {:?}", format),
+        }
+    }
+}
+
+impl Into<image::ImageFormat> for ImageFormat {
+    fn into(self) -> image::ImageFormat {
+        match self {
+            ImageFormat::PNG => image::ImageFormat::PNG,
+            ImageFormat::JPEG => image::ImageFormat::JPEG,
+            ImageFormat::GIF => image::ImageFormat::GIF,
+            ImageFormat::WEBP => image::ImageFormat::WEBP,
+            ImageFormat::TIFF => image::ImageFormat::TIFF,
+            ImageFormat::BMP => image::ImageFormat::BMP,
+            ImageFormat::ICO => image::ImageFormat::ICO,
+        }
+    }
+}
+
+pub struct Image {
+    image: DynamicImage,
+    format: ImageFormat,
+}
+
+impl Image {
+    #[inline]
+    pub fn mime(&self) -> &str {
+        self.format.mime()
+    }
+
+    #[inline]
+    pub fn width(&self) -> u32 {
+        self.image.width()
+    }
+
+    #[inline]
+    pub fn height(&self) -> u32 {
+        self.image.height()
+    }
+}
+
+impl AsRef<DynamicImage> for Image {
+    fn as_ref(&self) -> &DynamicImage {
+        &self.image
+    }
+}
+
+#[inline]
+pub fn guess_format(buf: &[u8]) -> Result<ImageFormat> {
+    let format = image::guess_format(buf)?;
+    ImageFormat::try_from(&format)
+}
+
+pub fn load(buf: &[u8]) -> Result<Image> {
+    let img_format = image::guess_format(&buf)?;
+    let format = ImageFormat::try_from(&img_format)?;
+
+    let image = image::load_from_memory_with_format(&buf, img_format)?;
+
+    Ok(Image {
+        image,
+        format,
+    })
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+
+    #[test]
+    fn verify_gfx_load_ico() {
+        let ico = fs::read("./sn0int-registry/assets/favicon.ico").expect("fs::read");
+        let img = load(&ico).expect("gfx::load");
+
+        assert_eq!("image/vnd.microsoft.icon", img.mime());
+        assert_eq!(16, img.height());
+        assert_eq!(16, img.width());
+    }
+}

src/keyring.rs

@@ -7,7 +7,7 @@ use crate::paths;
 use std::collections::{HashMap, HashSet};
 use std::fs;
 use std::str::FromStr;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use sn0int_common::ModuleID;
 
 
@@ -85,7 +85,7 @@ impl KeyRing {
         }
     }
 
-    pub fn load(path: &PathBuf) -> Result<KeyRing> {
+    pub fn load(path: &Path) -> Result<KeyRing> {
         let buf = fs::read(&path)?;
         serde_json::from_slice(&buf)
             .map_err(Error::from)
@@ -202,6 +202,17 @@ impl KeyRingEntry {
         let v = LuaJsonValue::from(v).into();
         Ok(v)
     }
+
+    pub fn matches(&self, query: &str) -> bool {
+        if let Some(idx) = query.find(':') {
+            let (namespace, access_key) = query.split_at(idx);
+            let access_key = &access_key[1..];
+
+            self.namespace == namespace && self.access_key == access_key
+        } else {
+            self.namespace == query
+        }
+    }
 }
 
 #[cfg(test)]

src/lib.rs

@@ -18,6 +18,7 @@ pub mod api;
 pub mod archive;
 pub mod args;
 pub mod auth;
+pub mod blobs;
 pub mod cmd;
 pub mod complete;
 pub mod config;
@@ -25,8 +26,10 @@ pub mod crt;
 pub mod db;
 pub mod errors;
 pub mod engine;
+pub mod filters;
 pub mod fmt;
 pub mod geoip;
+pub mod gfx;
 pub mod html;
 pub mod json;
 pub mod keyring;
@@ -43,7 +46,15 @@ pub mod ser;
 pub mod shell;
 pub mod sockets;
 pub mod term;
+pub mod update;
 pub mod utils;
 pub mod web;
 pub mod worker;
 pub mod workspaces;
+
+
+#[cfg(test)]
+fn test_init() {
+    use env_logger;
+    let _ = env_logger::builder().is_test(true).try_init();
+}

src/main.rs

@@ -1,8 +1,8 @@
 #![warn(unused_extern_crates)]
-use env_logger;
 #[macro_use] extern crate failure;
+#[macro_use] extern crate log;
 
-use env_logger::Env;
+use env_logger::{self, Env};
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
 use sn0int::cmd;
@@ -22,27 +22,25 @@ use std::io::Write;
 use std::path::Path;
 
 
-fn run_run(gargs: &Args, args: &args::Run, config: Config) -> Result<()> {
+fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
     let mut rl = shell::init(gargs, config, false)?;
 
-    if let Some(module) = &args.module {
-        let module = rl.engine().get(&module)?.clone();
-        rl.set_module(module);
-    } else if let Some(file) = &args.file {
-        let path = Path::new(file);
+    let module = if args.file {
+        let path = Path::new(&args.module);
 
         let filename = path.file_stem()
             .ok_or(format_err!("Failed to decode filename"))?
             .to_str()
             .ok_or(format_err!("Failed to decode filename"))?;
 
-        let module = Module::load(&path.to_path_buf(), "anonymous", &filename)
-            .context(format!("Failed to parse {:?}", file))?;
-        rl.set_module(module);
+        Module::load(&path.to_path_buf(), "anonymous", &filename, true)
+            .context(format!("Failed to parse {:?}", path))?
     } else {
-        bail!("At least one module or file need to be provided");
-    }
+        rl.engine().get(&args.module)?
+            .clone()
+    };
 
+    rl.set_module(module);
     cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
 }
 
@@ -56,7 +54,7 @@ fn run_sandbox() -> Result<()> {
     engine::isolation::run_worker(geoip, asn, &psl)
 }
 
-fn run_cmd<T: cmd::Cmd>(gargs: &Args, args: &T, config: Config) -> Result<()> {
+fn run_cmd<T: cmd::Cmd>(gargs: &Args, args: T, config: &Config) -> Result<()> {
     let mut rl = shell::init(gargs, config, false)?;
     args.run(&mut rl)
 }
@@ -81,7 +79,7 @@ end
 }
 
 fn run() -> Result<()> {
-    let args = Args::from_args();
+    let mut args = Args::from_args();
 
     if !args.is_sandbox() {
         sandbox::fasten_seatbelt()?;
@@ -90,17 +88,22 @@ fn run() -> Result<()> {
     let config = Config::load_or_default()
         .context("Failed to load config")?;
 
-    match args.subcommand {
-        Some(SubCommand::Run(ref run)) => run_run(&args, run, config),
+    debug!("Loaded config: {:?}", config);
+
+    match args.subcommand.take() {
+        Some(SubCommand::Run(run)) => run_run(&args, &run, &config),
         Some(SubCommand::Sandbox(_)) => run_sandbox(),
         Some(SubCommand::Login(_)) => auth::run_login(&config),
-        Some(SubCommand::New(ref new)) => run_new(&args, new),
-        Some(SubCommand::Publish(ref publish)) => registry::run_publish(&args, publish, &config),
-        Some(SubCommand::Install(ref install)) => registry::run_install(install, &config),
-        Some(SubCommand::Search(ref search)) => registry::run_search(search, &config),
-        Some(SubCommand::Select(ref select)) => run_cmd(&args, select, config),
-        Some(SubCommand::Completions(ref completions)) => complete::run_generate(completions),
-        None => shell::run(&args, config),
+        Some(SubCommand::New(new)) => run_new(&args, &new),
+        Some(SubCommand::Publish(publish)) => registry::run_publish(&args, &publish, &config),
+        Some(SubCommand::Install(install)) => registry::run_install(&install, &config),
+        Some(SubCommand::Search(search)) => registry::run_search(&search, &config),
+        Some(SubCommand::Add(add)) => run_cmd(&args, add, &config),
+        Some(SubCommand::Select(select)) => run_cmd(&args, select, &config),
+        Some(SubCommand::Delete(delete)) => run_cmd(&args, delete, &config),
+        Some(SubCommand::Fsck(fsck)) => run_cmd(&args, fsck, &config),
+        Some(SubCommand::Completions(completions)) => complete::run_generate(&completions),
+        None => shell::run(&args, &config),
     }
 }