Release v0.19.1

Fix early exit on notification and make ratelimiter more global

.dockerignore

@@ -1,6 +1,7 @@
 target
 Dockerfile
 .dockerignore
+docker-compose.yml
 docker
 docs
 ci

.travis.yml

@@ -5,18 +5,6 @@ language: rust
 
 matrix:
   include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
   - os: linux
     rust: stable
     env:
@@ -25,6 +13,22 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=docker-registry
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=test
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=common
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=std
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=boxxy
   - os: osx
     rust: stable
     env:
@@ -33,14 +37,18 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=common
+  - os: osx
+    rust: stable
+    env:
+    - BUILD_MODE=std
   #- os: windows
   #  rust: stable
   #  env:
   #  - BUILD_MODE="windows test"
-  - os: windows
-    rust: stable
-    env:
-    - BUILD_MODE="windows common"
+  #- os: windows
+  #  rust: stable
+  #  env:
+  #  - BUILD_MODE="windows common"
 
 before_install:
 - ci/setup.sh "$TRAVIS_OS_NAME"
@@ -51,8 +59,11 @@ script:
 
 notifications:
   irc:
+    template:
+    - "%{repository_slug}#%{build_number} (%{branch} - %{commit} : %{author}): %{message} (%{build_url})"
     channels:
     - "ircs://irc.hackint.org:6697/#sn0int"
+    use_notice: true
+    skip_join: true
     #on_success: change # default: always
     #on_failure: always # default: always
-    use_notice: true

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.14.0"
+version = "0.19.1"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -13,15 +13,32 @@ edition = "2018"
 travis-ci = { repository = "kpcyrd/sn0int" }
 
 [workspace]
-members = ["sn0int-registry/sn0int-common",
-           "sn0int-registry"]
+members = ["sn0int-common",
+           "sn0int-registry",
+           "sn0int-std"]
+
+[package.metadata.deb]
+extended-description = """\
+sn0int (pronounced /snoɪnt/) is a semi-automatic OSINT framework and package
+manager. It was built for IT security professionals and bug hunters to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations."""
+section = "utils"
+priority = "optional"
+depends = "$auto, publicsuffix"
+build-depends = "pkg-config, libsqlite3-dev, libseccomp-dev, libsodium-dev, publicsuffix"
+assets = [
+    ["target/release/sn0int", "usr/bin/", "755"],
+]
 
 [features]
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.9.0", path="sn0int-registry/sn0int-common" }
-rustyline = "5.0"
+sn0int-common = { version="0.11.0", path="sn0int-common" }
+sn0int-std = { version="=0.19.0", path="sn0int-std" }
+rustyline = "6.0"
 log = "0.4"
 env_logger = "0.7"
 hlua-badtouch = "0.4"
@@ -31,23 +48,17 @@ rand = "0.7"
 colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
-publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-libsqlite3-sys = "0.16.0"
+libsqlite3-sys = { version = "0.17.0", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
 dirs = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.13"
-rustls = { version="0.16", features=["dangerous_configuration"] }
-webpki = "0.21"
-webpki-roots = "0.18"
-pem = "0.7"
-base64 = "0.11"
+chrootable-https = "0.15.2"
+base64 = "0.12"
 data-encoding = "2.1.2"
-kuchiki = "0.7.2"
 serde_urlencoded = "0.6"
 serde = "1.0"
 serde_derive = "1.0"
@@ -60,45 +71,36 @@ maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.5"
 regex = "1.0"
 toml = "0.5"
-maxminddb = "0.13"
-tar = "0.4.17"
-libflate = "0.1.14"
 threadpool = "1.7"
-x509-parser = "0.6.0"
-der-parser = "3.0"
 atty = "0.2"
-bufstream = "0.1.4"
-tokio = "0.1.14"
 semver = "0.9"
 bytes = "0.4"
-xml-rs = "0.8"
 bytesize = "1.0"
-ipnetwork = "0.15"
-strum = "0.16"
-strum_macros = "0.16"
+ipnetwork = "0.16"
+strum = "0.18"
+strum_macros = "0.18"
+embedded-triple = "0.1.0"
 
 digest = "0.8.0"
-bs58 = "0.3"
-blake2 = "0.8.0"
 md-5 = "0.8.0"
 sha-1 = "0.8.1"
 sha2 = "0.8.0"
 sha3 = "0.8.0"
 hmac = "0.7"
 
-image = "0.22"
-kamadak-exif = "0.3.1"
 walkdir = "2.2"
-nude = "0.2"
+nude = "0.3"
+glob = "0.3.0"
+os-version = "0.1.1"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.11"
-nix = "0.15"
+syscallz = "0.14.0"
+nix = "0.17"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
-pledge = "0.3.1"
+pledge = "0.4"
 unveil = "0.2.0"
 
 [dev-dependencies]

Dockerfile

@@ -1,14 +1,13 @@
-FROM rust:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

Makefile

@@ -1,19 +1,23 @@
 check:
-	(cd sn0int-registry/sn0int-common; cargo check)
+	(cd sn0int-common; cargo check)
 	(cd sn0int-registry; cargo check)
+	(cd sn0int-std; cargo check)
 	cargo check
 
 force-check:
-	(cd sn0int-registry/sn0int-common; touch src/lib.rs; cargo check)
+	(cd sn0int-common; touch src/lib.rs; cargo check)
 	(cd sn0int-registry; touch src/main.rs; cargo check)
+	(cd sn0int-std; touch src/lib.rs; cargo check)
 	touch src/lib.rs
 	cargo check
 
 test:
-	(cd sn0int-registry/sn0int-common; cargo test)
+	(cd sn0int-common; cargo test)
 	(cd sn0int-registry; cargo test)
-	cargo test
-	cargo test -- --ignored
+	(cd sn0int-std; cargo test)
+	(cd sn0int-std; cargo test -- --ignored)
+	cargo test --lib
+	cargo test --lib -- --ignored
 
 update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt

README.md

@@ -58,6 +58,39 @@ Mac OSX
 
     brew install sn0int
 
+Debian/Ubuntu/Kali
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+    apt install debian-keyring
+    gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | apt-key add -
+    apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    echo deb http://apt.vulns.sexy stable main > /etc/apt/sources.list.d/apt-vulns-sexy.list
+    apt update
+    apt install sn0int
+
+Docker
+
+    docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
+
+Alpine
+
+    apk add sn0int
+
+OpenBSD
+
+    pkg_add sn0int
+
+Gentoo
+
+    layman -a pentoo
+    emerge --ask net-analyzer/sn0int
+
+NixOS
+
+    nix-env -i sn0int
+
 For everything else please have a look at the [detailed list][1].
 
 [1]: https://sn0int.readthedocs.io/en/latest/install.html
@@ -68,10 +101,12 @@ For everything else please have a look at the [detailed list][1].
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
   - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
+  - [Fedora/CentOS/Redhat](https://sn0int.readthedocs.io/en/latest/install.html#fedora-centos-redhat)
   - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
   - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
   - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
+  - [NixOS](https://sn0int.readthedocs.io/en/latest/install.html#nixos)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
 - [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
   - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
@@ -89,10 +124,10 @@ For everything else please have a look at the [detailed list][1].
   - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
   - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
   - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
-  - [Autonoscope](https://sn0int.readthedocs.io/en/latest/usage.html#autonoscope)
-    - [Domains](https://sn0int.readthedocs.io/en/latest/usage.html#domains)
-    - [IPs](https://sn0int.readthedocs.io/en/latest/usage.html#ips)
-    - [URLs](https://sn0int.readthedocs.io/en/latest/usage.html#urls)
+- [Autonoscope](https://sn0int.readthedocs.io/en/latest/autonoscope.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/autonoscope.html#domains)
+  - [IPs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#ips)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#urls)
 - [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
   - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
@@ -100,6 +135,8 @@ For everything else please have a look at the [detailed list][1].
   - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/database.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/database.html#db-activity)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
 - [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
@@ -116,10 +153,29 @@ For everything else please have a look at the [detailed list][1].
   - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
   - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
   - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
+  - [CryptoAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#cryptoaddrs)
+  - [Activity](https://sn0int.readthedocs.io/en/latest/structs.html#activity)
   - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
     - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
     - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
     - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Activity](https://sn0int.readthedocs.io/en/latest/activity.html)
+  - [Anatomy of an event](https://sn0int.readthedocs.io/en/latest/activity.html#anatomy-of-an-event)
+  - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
+  - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
+  - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
+- [Notifications](https://sn0int.readthedocs.io/en/latest/notifications.html)
+  - [Receiving notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#receiving-notifications)
+    - [Telegram](https://sn0int.readthedocs.io/en/latest/notifications.html#telegram)
+    - [Pushover](https://sn0int.readthedocs.io/en/latest/notifications.html#pushover)
+    - [Discord](https://sn0int.readthedocs.io/en/latest/notifications.html#discord)
+    - [Signal](https://sn0int.readthedocs.io/en/latest/notifications.html#signal)
+    - [Writing your own module](https://sn0int.readthedocs.io/en/latest/notifications.html#writing-your-own-module)
+  - [Setting up notification rules](https://sn0int.readthedocs.io/en/latest/notifications.html#setting-up-notification-rules)
+  - [Testing notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#testing-notifications)
+  - [Running sn0int automatically](https://sn0int.readthedocs.io/en/latest/notifications.html#running-sn0int-automatically)
+    - [Monitors](https://sn0int.readthedocs.io/en/latest/notifications.html#monitors)
+    - [Timers](https://sn0int.readthedocs.io/en/latest/notifications.html#timers)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
@@ -147,6 +203,7 @@ For everything else please have a look at the [detailed list][1].
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/reference.html#db-activity)
   - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
@@ -170,16 +227,23 @@ For everything else please have a look at the [detailed list][1].
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [key_trunc_pad](https://sn0int.readthedocs.io/en/latest/reference.html#key-trunc-pad)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
   - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [mqtt_connect](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-connect)
+  - [mqtt_subscribe](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-subscribe)
+  - [mqtt_recv](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-recv)
+  - [mqtt_ping](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-ping)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [ratelimit_throttle](https://sn0int.readthedocs.io/en/latest/reference.html#ratelimit-throttle)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
   - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
@@ -195,6 +259,7 @@ For everything else please have a look at the [detailed list][1].
   - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
   - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
+  - [sock_options](https://sn0int.readthedocs.io/en/latest/reference.html#sock-options)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
   - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
   - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
@@ -206,11 +271,15 @@ For everything else please have a look at the [detailed list][1].
   - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
   - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [sodium_secretbox_open](https://sn0int.readthedocs.io/en/latest/reference.html#sodium-secretbox-open)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
   - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [str_find](https://sn0int.readthedocs.io/en/latest/reference.html#str-find)
+  - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
   - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
   - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [strval](https://sn0int.readthedocs.io/en/latest/reference.html#strval)
   - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
@@ -221,6 +290,14 @@ For everything else please have a look at the [detailed list][1].
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
   - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
   - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
+  - [ws_connect](https://sn0int.readthedocs.io/en/latest/reference.html#ws-connect)
+  - [ws_options](https://sn0int.readthedocs.io/en/latest/reference.html#ws-options)
+  - [ws_recv_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-text)
+  - [ws_recv_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-binary)
+  - [ws_recv_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-json)
+  - [ws_send_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-text)
+  - [ws_send_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-binary)
+  - [ws_send_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-json)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
   - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
   - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)

ci/integration.py

@@ -47,10 +47,10 @@ def main(tempdir, binary):
 
     print('[*] installing modules')
     sn0int(tempdir, binary, [
-        'mod install kpcyrd/ctlogs',
-        'mod install kpcyrd/dns-resolve',
-        'mod install kpcyrd/url-scan',
-        'mod install kpcyrd/geoip',
+        'pkg install kpcyrd/ctlogs',
+        'pkg install kpcyrd/dns-resolve',
+        'pkg install kpcyrd/url-scan',
+        'pkg install kpcyrd/geoip',
     ])
 
     print('[*] running ctlogs')
@@ -63,6 +63,7 @@ def main(tempdir, binary):
     print('[*] testing db for subdomains')
     subdomains = sn0int_select(tempdir, binary, ['subdomains'])
     assert {x['value'] for x in subdomains} == {
+        'example.com',
         'www.example.com',
         'm.example.com',
         'dev.example.com',
@@ -91,6 +92,8 @@ def main(tempdir, binary):
     print('[*] testing db for urls')
     urls = sn0int_select(tempdir, binary, ['urls'])
     assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
         ('http://www.example.com/', 200),
         ('https://www.example.com/', 200),
     }

ci/run.sh

@@ -6,21 +6,21 @@ case "$1" in
         cargo build --verbose --examples
         ;;
     test)
-        ci/run.sh build
-        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
-             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
         cargo test --verbose
         cargo test --verbose -- --ignored
         ;;
     common)
-        cd sn0int-registry/sn0int-common
+        cd sn0int-common
         cargo test --verbose
         ;;
+    std)
+        cd sn0int-std
+        cargo test --verbose
+        cargo test --verbose -- --ignored
+        ;;
     windows)
-        cargo build --verbose --features=sqlite-bundled
-        cargo build --verbose --examples --features=sqlite-bundled
+        cargo build --verbose
+        cargo build --verbose --examples
         ;;
     boxxy)
         cargo build --verbose --examples
@@ -35,7 +35,7 @@ case "$1" in
         docker run --rm sn0int --help
         ;;
     docker-registry)
-        docker build -t sn0int-registry sn0int-registry/
+        docker build -t sn0int-registry -f sn0int-registry/Dockerfile .
         docker images
         ;;
 esac

ci/setup.sh

@@ -3,6 +3,9 @@ set -exu
 case "$1" in
     linux)
         sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev
+        sudo apt install libsqlite3-dev libseccomp-dev libsodium-dev
+        ;;
+    osx)
+        brew install libsodium
         ;;
 esac

contrib/docker/Dockerfile.alpine

@@ -1,13 +1,13 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

contrib/docker/Dockerfile.debian

@@ -1,13 +1,13 @@
-FROM rust
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]

docs/activity.rst

@@ -0,0 +1,166 @@
+Activity
+========
+
+So far we've learned about regular `structs <structs.html>`_, but activity is
+special.
+
+Activity is an event tied to a specific time and topic and has a small amount
+of data piggybacked to it.
+
+Anatomy of an event
+-------------------
+
+``topic``
+    This is some freestyle text used to group events to a specific topic. This
+    must not conflict with other modules unless there's a very good reason.
+
+    The topic should look like ``kpcyrd/example:something``, with ``something``
+    being a meaningful unique identifier for whatever is generating these
+    events, like a mac address or an account name/id.
+
+    The rules around this might become stricter in the future.
+``time``
+    The most important part of the event: The time and date it happened.
+``initial``
+    This value can not be set but might be present in sn0int output. See `Querying events`_.
+``uniq`` (optional)
+    This is an optional feature to deduplicate events. Assuming you're
+    importing posts by an account, you wouldn't want to store a new event for
+    each post you already imported. If you set this field to the technical post
+    id then sn0int would skip the event if it already has an event with the
+    same ``topic`` and ``uniq`` combination to avoid inserting duplicates.
+``latitude`` (optional)
+    Latitude - if you can tie the event to a specific location.
+``longitude`` (optional)
+    Longitude - if you can tie the event to a specific location.
+``radius`` (optional)
+    The location radius in meters. If the position you got has a precision of
+    100 meters set this value to ``100``.
+``content``
+    Arbitrary data that you want to attach to the event. This doesn't need to
+    be a string and can be an arbitrary object that is then stored as json
+    string.
+
+Logging events
+--------------
+
+An ``activity`` event can be logged with ``db_activity``:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Logging an event that has a location attached could look like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        latitude=40.726662,
+        longitude=-74.036677,
+        radius=50,
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Making sure an event is not logged twice can be done with ``uniq``:
+
+.. code-block:: lua
+
+    -- create the first event
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this does nothing because we already have an event with this topic+uniq combination
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this creates a new event because uniq is different
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+    -- this also creates a new event because topic is different
+    db_activity({
+        topic='harness/activity-ping:something-else',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+Querying events
+---------------
+
+There is a commandline interface that can be used to query all events we've
+logged. To get everything (sorted by time)::
+
+    sn0int activity
+
+To limit the output to a specific topic::
+
+    sn0int activity -t harness/activity-ping:dummy
+
+To limit it to a specific time frame::
+
+    # everything since
+    sn0int activity --since 2020-01-13T04:20:00
+    # everything until
+    sn0int activity --until 2020-01-13T04:20:00
+    # both
+    sn0int activity --since yesterday --until today
+
+When using ``--since`` you might also want to know the previous state and use
+it as an initial value. Consider this example::
+
+    2020-01-13 14:30:00 # user goes offline
+    2020-01-13 23:59:00 # user goes online
+    2020-01-14 09:30:00 # user goes idle
+    2020-01-14 14:20:00 # user goes offline
+
+If we're running a query like ``sn0int activity --since 2020-01-14T00:00:00``
+the program consuming the output wouldn't know that the user is initially
+online because we're only getting this data::
+
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+We can tweak this with ``sn0int activity --initial --since
+2020-01-14T00:00:00`` to include one more event that we only use to populate
+the intial state::
+
+    {"id":7,"initial":true,"topic":"foo/bar:asdf","time":"2020-01-13T23:59:00","content":{"state":"online"}}
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+Visualization
+-------------
+
+There is no visualization built in, there may be external frontends for this in
+the future. You're very welcome to write one!

docs/autonoscope.rst

@@ -0,0 +1,81 @@
+Autonoscope
+===========
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+-------
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+---
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+----
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

docs/build.rst

@@ -18,19 +18,21 @@ Archlinux
 
 .. code-block:: bash
 
-    $ pacman -S geoip2-database libseccomp publicsuffix-list sqlite
+    $ pacman -S geoip2-database libseccomp libsodium publicsuffix-list sqlite
 
 Mac OSX
 ~~~~~~~
 
-None.
+.. code-block:: bash
+
+    $ brew install libsodium
 
 Debian/Ubuntu/Kali
 ~~~~~~~~~~~~~~~~~~
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
 
 .. warning::
    On a debian based system make sure you've installed rust with rustup.
@@ -40,21 +42,21 @@ Alpine
 
 .. code-block:: bash
 
-    $ apk add sqlite-dev libseccomp-dev
+    $ apk add sqlite-dev libseccomp-dev libsodium-dev
 
 OpenBSD
 ~~~~~~~
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3 geolite2-city geolite2-asn
+    $ pkg_add sqlite3 geolite2-city geolite2-asn libsodium
 
 Gentoo
 ~~~~~~
 
 .. code-block:: bash
 
-    emerge --ask sys-libs/libseccomp dev-db/sqlite
+    emerge --ask sys-libs/libseccomp dev-db/sqlite dev-libs/libsodium
 
 Windows
 ~~~~~~~
@@ -71,10 +73,4 @@ After all dependencies have been installed, simply build the binary:
 
     $ cargo build --release
 
-Note that you need a different command on windows:
-
-.. code-block:: bash
-
-    $ cargo build --release --features=sqlite-bundled
-
 After the build finished the binary is located at ``target/release/sn0int``.

docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'sn0int'
-copyright = '2018, kpcyrd'
+copyright = '2018-2020, kpcyrd'
 author = 'kpcyrd'
 
 # The short X.Y version

docs/database.rst

@@ -46,6 +46,45 @@ triggered and an db_update is performed instead.
    removed from scope with ``noscope``. Everytime you use ``db_add`` you need
    to make sure that the ID that has been returned is not ``nil``.
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add_ttl('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_update
 ---------
 

docs/index.rst

@@ -38,9 +38,12 @@ Getting Started
    install
    build
    usage
+   autonoscope
    scripting
    database
    structs
+   activity
+   notifications
    keyring
    config
    sandbox

docs/install.rst

@@ -22,9 +22,20 @@ Mac OSX
 Debian/Ubuntu/Kali
 ------------------
 
-Note that debian `doesn't ship the geoip2-database
-<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
-download them automatically during the first run.
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+.. code-block:: bash
+
+    $ apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | apt-key add -
+    $ apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    $ echo deb http://apt.vulns.sexy stable main > /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ apt update
+    $ apt install sn0int
+
+Fedora/CentOS/Redhat
+--------------------
 
 Using rust+cargo from the repos might work for you, but we only officially
 support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
@@ -32,7 +43,7 @@ at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ dnf install @development-tools libsq3-devel libseccomp-devel libsodium-devel publicsuffix-list
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .
@@ -47,8 +58,6 @@ Docker
 Alpine
 ------
 
-On alpine edge, with enabled testing repositories:
-
 .. code-block:: bash
 
     $ apk add sn0int
@@ -56,8 +65,6 @@ On alpine edge, with enabled testing repositories:
 OpenBSD
 -------
 
-On -current:
-
 .. code-block:: bash
 
     $ pkg_add sn0int
@@ -67,8 +74,15 @@ Gentoo
 
 .. code-block:: bash
 
-    layman -f -o https://raw.githubusercontent.com/kpcyrd/overlay/master/overlay.xml -a kpcyrd-overlay
-    emerge --ask net-analyzer/sn0int
+    $ layman -a pentoo
+    $ emerge --ask net-analyzer/sn0int
+
+NixOS
+-----
+
+.. code-block:: bash
+
+    $ nix-env -i sn0int
 
 Windows
 -------
@@ -82,4 +96,4 @@ Make sure rust is installed and setup.
 
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ cargo install -f --path . --features=sqlite-bundled
+    $ cargo install -f --path .

docs/keyring.rst

@@ -58,6 +58,10 @@ If the user granted us access to those keys we can read them with ``keyring``:
 This returns a list of all keys in that namespace. Any empty list is returned
 if the user doesn't have any keys in that namespace.
 
+If you want to allow the user to select a specific script you can introduce an
+option that is set by the user and then filter ``creds`` until the
+``access_key`` matches.
+
 Using access keys as source argument
 ------------------------------------
 

docs/notifications.rst

@@ -0,0 +1,311 @@
+Notifications
+=============
+
+If you run sn0int unattended nobody might see the sn0int output. For cases like
+this you can configure notifications to send you a push notification in case
+something interesting happens. This is also especially useful if you have
+sn0int setup to run automatically.
+
+Receiving notifications
+-----------------------
+
+Notifications are just regular sn0int modules. You can install them just like
+any other module or write your own. This section contains walkthroughs on how
+to setup common integrations.
+
+Telegram
+~~~~~~~~
+
+Install the telegram notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-telegram
+
+Open your telegram app and open a chat with ``@botfather``. Send ``/newbot``
+and answer the questions. Copy ``bot_token`` and open this url in your browser:
+
+.. code-block::
+
+    https://api.telegram.org/bot**your_bot_token**/getUpdates
+
+Back on your app, open the t.me link to start a new chat with your bot, then
+send ``/start``. Reload the page in your browser, you should see the new
+message you sent. Copy the ``chat_id``.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-telegram -o bot_token=1337:foobar -o chat_id=1337 'hello world'
+
+You should receive ``hello world`` from your bot on Telegram.
+
+Pushover
+~~~~~~~~
+
+Install the pushover notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-pushover
+
+Signup for pushover and configure the app on your device. Copy th user key
+visible on the pushover dashboard. Click "Create an Application/API Token". Set
+"sn0int" as name and set an icon if you want to. Copy the api token.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-pushover -o user_key=asdf1337 -o api_token=asdf1337 'hello world'
+
+You should receive ``hello world`` as a push notification.
+
+Discord
+~~~~~~~
+
+Install the discord notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-discord
+
+Decide which channel should receive notifications (or create a new one). Open
+the "Server Settings" of your discord server. Click on "Webhooks". Click
+"Create Webhook". Configure the Name and Channel. Copy the Webhook URL.
+
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-discord -o url=https://discord.com/api/webhooks/1337/asdf 'hello world'
+
+You should receive ``hello world`` in your discord channel.
+
+Signal
+~~~~~~
+
+Install the sn0int notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-signal
+
+This module allows end-to-end encrypted notifications, but it's also difficult
+to setup. You need a second phone number and install both `signal-cli
+<https://github.com/AsamK/signal-cli>`_ and `sn0int-signal
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+After you've registered your second phone number with signal-cli, you can use
+sn0int-signal to expose a minimal api for notify-signal. For more detailed
+instructions and how to start the api at boot, see the `sn0int-signal README
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+Read the secret key generated at ``/etc/sn0int-signal.key`` and send a
+notification to the signal phone number:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-signal -o to=+31337 -o secret=asdf 'hello world'
+
+You should receive ``hello world`` from the number signed up with signal-cli.
+
+Writing your own module
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Make sure you've read the detailed instructions on how to get setup with
+`module development <scripting.html>`_.
+
+Create a new sn0int module like this:
+
+.. code-block:: bash
+
+    sn0int new ~/repos/sn0int-modules/notify-custom.lua
+
+Edit the ``-- Source:`` so it takes notifications as input:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        debug(arg)
+        info(arg['subject'])
+        info(arg['body'])
+    end
+
+Execute your script:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom 'hello world'
+
+You most likely need to pass options to avoid hard-coding keys into your
+script. Options can be fetched like this:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        local foo = getopt('foo')
+        if not foo then return 'Missing -o foo= option' end
+
+        info('foo: ' .. foo)
+        info('subject: ' .. arg['subject'])
+    end
+
+And passed like this:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom -o "foo=hello world" 'ohai'
+
+Setting up notification rules
+-----------------------------
+
+We now know how to trigger notifications manually, but we would rather trigger
+notifications if a module runs into something interesting.
+
+You can setup subscriptions on specific topics and then have a notification
+script execute automatically.
+
+Lookup the location of your sn0int config file:
+
+.. code-block:: bash
+
+    sn0int paths
+
+And open it in an editor of your choice:
+
+.. code-block:: bash
+
+    vim /home/user/.config/sn0int.toml
+
+A basic configuration could look like this:
+
+.. code-block:: toml
+
+    # You can have multiple notification sections, this one is named
+    # `demo-telegram-integration`
+    # The label can be set to whatever you want, but you may need to add
+    # double-quotes to use some characters.
+    [notifications.demo-telegram-integration]
+    # If this option is present, the notification must originate from one of
+    # the following workspaces.
+    workspaces = ["default", "some-workspace"]
+    # If this option is present, the notification must match one of the
+    # filters. You can use `*` as a wildcard to match everything except `:`.
+    topics = ["activity:harness/activity-ping:*"]
+    # Mandatory: the module to execute.
+    script = "kpcyrd/notify-telegram"
+    # The options to pass to the module, if any.
+    # Can be accessed with `getopt`
+    options = [
+        "bot_token=1337:foobar",
+        "chat_id=1337",
+    ]
+
+All options except ``script`` are optional, but setting filters is highly
+recommended.
+
+Testing notifications
+---------------------
+
+To test if your configuration works correctly you can create an event manually:
+
+.. code-block:: bash
+
+    sn0int -w some-workspace notify send activity:harness/activity-ping:dummy "hello world"
+
+If it matches any of your rules you should receive a push notifications.
+
+.. note::
+    If you want to test just the routing without actually sending something, add ``--dry-run``.
+
+Running sn0int automatically
+----------------------------
+
+Support for this is going to improve in the future, but you can already set
+this up if you're ok with a slightly buggy experience.
+
+Monitors
+~~~~~~~~
+
+Some modules are long-running and either wait for an event from a server or
+have custom polling built in that's usually configurable with an ``-o
+interval=`` option. If your module has a non-trivial setup phase, an author may
+take this approach.
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-new-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+    Restart=always
+    RestartSec=0
+
+    [Install]
+    WantedBy=multi-user.target
+
+Enable the service to run on boot:
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-new-service.service
+
+Timers
+~~~~~~
+
+If the module is only one-shot you can set it up to run with a timer:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+Setup the timer like this:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.timer
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Timer]
+    OnBootSec=1min
+    OnUnitActiveSec=1h
+
+    [Install]
+    WantedBy=timers.target
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-other-service.timer

docs/reference.rst

@@ -153,6 +153,28 @@ ttl.
         device_id=13,
     }, 120)
 
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_select
 ---------
 
@@ -369,6 +391,8 @@ options are set. The following options are available:
 ``proxy``
   Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
   if it doesn't conflict with the global proxy settings.
+``binary``
+  Set to ``true`` to get the http response as raw bytes.
 
 This function may fail.
 
@@ -396,6 +420,8 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``binary``
+  The response body as bytes (if ``binary=true``)
 ``blob``
   If ``into_blob`` was enabled for the request the body is downloaded into blob
   storage with a reference to the body in this field.
@@ -495,6 +521,13 @@ Log an info to the terminal.
 
     info('ohai')
 
+intval
+------
+
+Parse a number from a string.
+
+    x = strval('1234')
+
 json_decode
 -----------
 
@@ -529,6 +562,17 @@ Encode a datastructure into a string.
     })
     print(x)
 
+key_trunc_pad
+-------------
+
+Truncate/pad a key to a given length.
+
+.. code-block:: lua
+
+    -- if longer than 32 bytes: truncate to 32
+    -- if shorter than 32 bytes: pad with \x00
+    local key = key_trunc_pad(password, 32, 0)
+
 keyring
 -------
 
@@ -563,6 +607,64 @@ Hash a byte array with md5 and return the results as bytes.
 
     hex(md5("\x00\xff"))
 
+mqtt_connect
+------------
+
+Connect to an mqtt broker.
+
+.. code-block:: lua
+
+    local sock = mqtt_connect('mqtts://mqtt.example.com', {
+        username='foo',
+        password='secret',
+    })
+    if last_err() then return end
+
+mqtt_subscribe
+--------------
+
+Subscribe to a topic. Right now only QoS 0 is supported.
+
+.. code-block:: lua
+
+    mqtt_subscribe(sock, '#', 0)
+    if last_err() then return end
+
+mqtt_recv
+---------
+
+Receive an mqtt packet. This is not necessarily a publish packet and more
+packets might be added in the future, so you need to check the type
+specifically.
+
+If a read timeout has been set with mqtt_connect_ this function returns ``nil``
+in case of a read timeout.
+
+.. code-block:: lua
+
+    local pkt = mqtt_recv(sock)
+    if last_err() then return end
+    if pkt == nil then
+        -- read timeout, consider sending a ping or disconnect if the previous ping failed
+    elseif pkt['type'] == 'pong' then
+        -- broker sent a pong
+    elseif pkt['type'] == 'publish' then
+        local payload = utf8_decode(pkt['body'])
+        if last_err() then return end
+        info(payload)
+    end
+
+mqtt_ping
+---------
+
+Send a pingreq packet, causing the broker to send a pingresp. This is used to
+make sure the connection is still working correctly.
+
+.. code-block:: lua
+
+    mqtt_ping(sock)
+    if last_err() then return end
+
 pgp_pubkey
 ----------
 
@@ -643,6 +745,22 @@ Returns the parent domain according to the public suffix list. For
     domain = psl_domain_from_dns_name('www.a.b.c.d.example.co.uk')
     print(domain == 'example.co.uk')
 
+ratelimit_throttle
+------------------
+
+Create a ratelimit that can only be passed x times every y milliseconds. This
+limit is global for a single ``run`` and also works with threads.
+
+.. code-block:: lua
+
+    -- allow this to pass every 250ms
+    ratelimit_throttle('foo', 1, 250)
+    -- allow this to pass not more than 4 times per second
+    ratelimit_throttle('foo', 4, 1000)
+
+This is useful if you need to coordinate your executions to stay below a
+certain request threshold.
+
 regex_find
 ----------
 
@@ -818,6 +936,13 @@ The following options are available:
 ``proxy``
   Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
   if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
 
 .. code-block:: lua
 
@@ -849,6 +974,23 @@ discarded when using sock_connect_ directly with ``tls=true``.
 
     info(tls)
 
+sock_options
+------------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock_options(sock, {
+        read_timeout=3,
+    })
+
 sock_send
 ---------
 
@@ -950,6 +1092,27 @@ Overwrite the default ``\n`` newline.
 
     sock_newline(sock, "\r\n")
 
+sodium_secretbox_open
+---------------------
+
+Use authenticated symetric crypto to decrypt a given message.
+
+Internally this is ``crypto_secretbox_xsalsa20poly1305``.
+
+The key **must** be 32 bytes, see key_trunc_pad_ if necessary.
+
+The first 24 bytes of the encrypted message are expected to be the nonce.
+
+.. code-block:: lua
+
+    plain = sodium_secretbox_open(encrypted, key)
+    if last_err() then return end
+
+    txt = utf8_decode(plain)
+    if last_err() then return end
+
+    info(txt)
+
 status
 ------
 
@@ -985,6 +1148,32 @@ Read stdin until EOF as a utf-8 string.
 .. note::
    This only works with `sn0int run --stdin`.
 
+str_find
+--------
+
+Returns the byte index of the first character that matches the pattern. This is
+explicitly a literal match instead of a lua pattern.
+
+If no match is found, returns ``nil``.
+
+.. code-block:: lua
+
+    x = str_find('asdf', 'sd')
+    print(x == 2)
+
+str_replace
+-----------
+
+Replaces all matches of a pattern in a string. This is explicitly a literal
+match instead of a lua pattern.
+
+If no match is found, an unmodified copy is returned.
+
+.. code-block:: lua
+
+    x = str_replace('this is old', 'old', 'new')
+    print(x == 'this is new')
+
 strftime
 --------
 
@@ -1005,6 +1194,13 @@ Parse a date into a unix timestamp, see `strftime rules`_.
 
 .. _strftime rules: https://docs.rs/chrono/0.4.6/chrono/format/strftime/index.html
 
+strval
+------
+
+Convert a number into a string.
+
+    x = strval(1234)
+
 time_unix
 ---------
 
@@ -1125,6 +1321,111 @@ a ``run`` execution.
     warn_once('ohai')
     warn_once('ohai')
 
+ws_connect
+----------
+
+Create a websocket connection. The url format is ``ws://example.com/asdf``,
+``wss://`` is also supported.
+
+The following options are available:
+
+``headers``
+  A map of additional headers that should be set for the request.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock = ws_connect("wss://example.com/asdf", {})
+
+ws_options
+----------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    ws_options(sock, {
+        read_timeout=3,
+    })
+
+ws_recv_text
+------------
+
+Wait until the server sends a text frame. A binary frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_text(sock)
+
+ws_recv_binary
+--------------
+
+Wait until the server sends a binary frame. A text frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_binary(sock)
+
+ws_recv_json
+------------
+
+Identical to ws_send_text_ but automatically runs json_decode_ on the
+response.
+
+.. code-block:: lua
+
+    msg = ws_recv_json(sock)
+
+ws_send_text
+------------
+
+Send a text frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, "ohai!")
+
+ws_send_binary
+--------------
+
+Send a binary frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_binary(sock, "\x00\x01\x02")
+
+ws_send_json
+------------
+
+Encode the object as json string and send it as a text frame on the websocket
+connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, {
+        foo="ohai!",
+        x={
+            y={1,3,3,7},
+        },
+    })
+
 x509_parse_pem
 --------------
 

docs/sandbox.rst

@@ -6,7 +6,7 @@ child process. It's important to note that there's a basic sandbox that's
 active on every operating system, and there's a second line of defense on
 supported operating systems.
 
-The first line of defense is the restrictive stdlib. It's assumed that and
+The first line of defense is the restrictive stdlib. It's assumed that an
 attacker gains full control over the lua code and is able to call any function
 with arbitrary arguments. The stdlib only provides functions that are
 considered safe, so for example it's not possible to start a process or open a

docs/structs.rst

@@ -259,6 +259,35 @@ is ``140.82.112.0/20``.
     This field isn't strictly defined and meant to be used as a human
     meaningful name if available.
 
+CryptoAddrs
+-----------
+
+A cryptoaddr is any cryptocurrency address and not tied to a specific currency.
+
+``value``
+    The address string. This looks like ``1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2``.
+``currency``
+    The identifier for a specific currency. This is usually the ticker symbols,
+    like ``xbt``, ``zec`` or ``xmr``.
+``denominator``
+    Balance is tracked internally using 64 bit integers (signed, for technical reasons). Balance is supposed to be the lowest unit, so in case of bitcoin you'd write ``100,000,000`` satoshi instead of ``1`` bitcoin. Since this value is inconvinient to work with we're using the denominator to display values. In case of bitcoin you'd set it to ``8``.
+``balance``
+    The current balance of the address, in the lowest possible unit. In case of bitcoin this would be satoshis.
+``received``
+    The total amount of currency received by this address.
+``first_seen``
+    The first time currency was sent to this address.
+``last_withdrawal``
+    The last time a transaction signed by this address was observed.
+``description``
+    A human readable note for this address.
+
+Activity
+--------
+
+Activity is different from all other structs, have a look at the `Activity
+Section <activity.html>`_.
+
 Relations
 ---------
 

docs/usage.rst

@@ -27,12 +27,12 @@ number of recommended modules::
     [+] Downloading "GeoLite2-City.mmdb"
     [+] Downloading "GeoLite2-ASN.mmdb"
     [+] Loaded 0 modules
-    [*] No modules found, run quickstart to install default modules
+    [*] No modules found, run pkg quickstart to install default modules
     [sn0int][default] >
 
-Typing ``quickstart`` is going to get you a fair number of featured modules::
+Typing ``pkg quickstart`` is going to get you a fair number of featured modules::
 
-    [sn0int][default] > quickstart
+    [sn0int][default] > pkg quickstart
     [+] Installing kpcyrd/asn
     [+] Installing kpcyrd/ctlogs
     [+] Installing kpcyrd/dns-resolve
@@ -105,7 +105,7 @@ Running a module
 Now that we have something to get started with, we can run our first module.
 First lets list all modules we have::
 
-    [sn0int][demo] > mod list
+    [sn0int][demo] > pkg list
     kpcyrd/asn (0.1.0)
         Run a asn lookup for an ip address
     kpcyrd/ctlogs (0.1.0)
@@ -149,7 +149,7 @@ some of them in a browser but hold on, there's a more efficient way to approach
 this.
 
 .. hint::
-   You can run the modules concurrently with ``run -j 8``.
+   You can run the modules concurrently with ``run -j3``.
 
 Running followup modules on the results
 ---------------------------------------
@@ -256,85 +256,3 @@ You can reverse this using the scope command::
 .. hint::
    All entities have this field, you can refer to it in queries using
    ``unscoped=1``.
-
-Autonoscope
------------
-
-Instead of manually unscoping everything you can also define so called
-autonoscope rules. Those are executed from most specific to least specific and
-the first match wins. If no rule matches, the default is in-scope::
-
-    [sn0int][demo] > # add the domain first
-    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
-    [sn0int][demo] > add domain example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > # automatically noscope all subdomains
-    [sn0int][demo] > autonoscope add domain example.com
-    [sn0int][demo] > # except subdomains of prod.example.com 
-    [sn0int][demo] > autoscope add domain prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > autonoscope list 
-      scope domain "prod.example.com"
-    noscope domain "example.com"
-    [sn0int][demo] > 
-    [sn0int][demo] > # this is going to be out-of-scope
-    [sn0int][demo] > add subdomain www.example.com
-    [sn0int][demo] > # this is going to be in-scope
-    [sn0int][demo] > add subdomain db.prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > select subdomains 
-    #1, "www.example.com"
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=0
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=1
-    #1, "www.example.com"
-    [sn0int][demo] > 
-
-Domains
-~~~~~~~
-
-Autonoscope rules for domains are applied to the following structs:
-
-- domains
-- subdomains
-- urls
-
-Example rules::
-
-    autonoscope add domain example.com
-    autonoscope add domain staging.example.com
-    autonoscope add domain com
-    autonoscope add domain .
-
-IPs
-~~~
-
-Autonoscope rules for IPs are applied to the following structs:
-
-- ipaddrs
-- netblocks
-- ports
-
-Example rules::
-
-    autonoscope add ip 0.0.0.0/0
-    autonoscope add ip ::/0
-    autonoscope add ip 192.168.0.0/16
-    autonoscope add ip 10.13.33.37/32
-
-URLs
-~~~~
-
-Autonoscope rules for urls are applied to the following structs:
-
-- urls
-
-Note that these rules are specific to a certain origin (like
-``https://example.com``) and are used to filter paths.
-
-Example rules::
-
-    autonoscope add url https://example.com/
-    autonoscope add url https://example.com/admin/
-    autonoscope add url https://example.com/a/b/c/d

examples/maxmind.rs

@@ -1,61 +1,27 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
 use sn0int::paths;
-use std::fs;
 use std::net::IpAddr;
+use std::path::Path;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
 pub enum Args {
-    #[structopt(name="dl")]
-    Download(Download),
     #[structopt(name="asn")]
     Asn(AsnArgs),
     #[structopt(name="geoip")]
     GeoIP(GeoIPArgs),
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Download {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-impl Download {
-    fn run(&self) -> Result<()> {
-        let path = paths::cache_dir()?.join(&self.target);
-        if self.extract_only {
-            let body = fs::read(&self.url)?;
-            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
-        } else {
-            GeoIP::download(path, &self.filter, &self.url)?;
-        }
-        Ok(())
-    }
-}
-
 #[derive(Debug, StructOpt)]
 pub struct AsnArgs {
     ip: IpAddr,
 }
 
 impl AsnArgs {
-    fn run(&self) -> Result<()> {
-        let asndb = AsnDB::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = AsnDB::cache_path(cache_dir)?;
+        let asndb = AsnDB::open(&path)?;
 
         let asn = asndb.lookup(self.ip)?;
         println!("{:#?}", asn);
@@ -70,8 +36,9 @@ pub struct GeoIPArgs {
 }
 
 impl GeoIPArgs {
-    fn run(&self) -> Result<()> {
-        let geoip = GeoIP::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = GeoIP::cache_path(cache_dir)?;
+        let geoip = GeoIP::open(&path)?;
 
         let lookup = geoip.lookup(self.ip)?;
         println!("{:#?}", lookup);
@@ -84,10 +51,10 @@ impl GeoIPArgs {
 fn run() -> Result<()> {
     let args = Args::from_args();
     debug!("{:?}", args);
+    let cache_dir = paths::cache_dir()?;
     match args {
-        Args::Download(args) => args.run(),
-        Args::Asn(args) => args.run(),
-        Args::GeoIP(args) => args.run(),
+        Args::Asn(args) => args.run(&cache_dir),
+        Args::GeoIP(args) => args.run(&cache_dir),
     }
 }
 

migrations/2020-01-09-024234_activity/down.sql

@@ -0,0 +1,31 @@
+DROP TABLE activity;
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-01-09-024234_activity/up.sql

@@ -0,0 +1,44 @@
+CREATE TABLE activity (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    topic VARCHAR NOT NULL,
+    time DATETIME NOT NULL,
+    uniq VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    radius INTEGER,
+    content VARCHAR NOT NULL
+);
+CREATE UNIQUE INDEX activity_uniq ON activity(topic, uniq);
+CREATE INDEX activity_topic ON activity(topic);
+CREATE INDEX activity_time ON activity(time);
+CREATE INDEX activity_topic_time ON activity(topic, time);
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, expire)
+    SELECT id, family, key, expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/up.sql

@@ -0,0 +1,19 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, value, expire)
+    SELECT id, family, key, "unknown", expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

modules/harness/activity-ping-once.lua

@@ -0,0 +1,26 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    db_activity({
+        topic=topic,
+        time=sn0int_time(),
+        uniq=uniq,
+        latitude=lat,
+        longitude=lon,
+        radius=radius,
+        content={
+            msg='ohai',
+        },
+    })
+end

modules/harness/activity-ping.lua

@@ -0,0 +1,29 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    while true do
+        db_activity({
+            topic=topic,
+            time=sn0int_time(),
+            uniq=uniq,
+            latitude=lat,
+            longitude=lon,
+            radius=radius,
+            content={
+                msg='ohai',
+            },
+        })
+        sleep(5)
+    end
+end

modules/harness/commit-log.lua

@@ -0,0 +1,26 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- git log -s --format='%H %ci'
+
+function run()
+    while true do
+        local x = stdin_readline()
+        if x == nil then
+            break
+        end
+        local m = regex_find('^(\\S+) (.+)', x)
+        if m then
+            time = strptime('%Y-%m-%d %T %z', m[3])
+            time = sn0int_time_from(time)
+
+            db_activity({
+                topic='harness/sn0int-commit:dummy',
+                time=time,
+                uniq=m[2],
+                content={},
+            })
+        end
+    end
+end

modules/harness/dummy-resolve.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: subdomains
+
+function run(arg)
+    db_update('subdomain', arg, {
+        resolvable=true,
+    })
+end

modules/harness/dummy-subdomains.lua

@@ -0,0 +1,13 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    for i=1, 20 do
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=http_mksession() .. '.' .. arg['value'],
+        })
+    end
+end

modules/harness/geo-polygon-contains.lua

@@ -0,0 +1,74 @@
+-- Description: demonstrate geofencing with polygons
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    hamburg = {
+        { lat=53.63975308945899,  lon=9.764785766601562  },
+        { lat=53.59494998253459,  lon=9.827270507812     },
+        { lat=53.663153974456456, lon=9.9151611328125    },
+        { lat=53.65582987649682,  lon=9.976272583007812  },
+        { lat=53.68613523817129,  lon=9.992752075195312  },
+        { lat=53.68674518938816,  lon=10.051460266113281 },
+        { lat=53.72495117617815,  lon=10.075492858886719 },
+        { lat=53.71946627930625,  lon=10.118408203125    },
+        { lat=53.743635083157756, lon=10.164413452148438 },
+        { lat=53.73104466704585,  lon=10.202865600585938 },
+        { lat=53.676781546441546, lon=10.16304016113281  },
+        { lat=53.632832079199474, lon=10.235824584960938 },
+        { lat=53.608803292930894, lon=10.2008056640625   },
+        { lat=53.578646152866504, lon=10.208358764648438 },
+        { lat=53.57212285981298,  lon=10.163726806640625 },
+        { lat=53.52071674896369,  lon=10.18707275390625  },
+        { lat=53.52643162253097,  lon=10.224151611328125 },
+        { lat=53.44062753992289,  lon=10.347747802734375 },
+        { lat=53.38824275010831,  lon=10.248870849609375 },
+        { lat=53.38824275010831,  lon=10.15960693359375  },
+        { lat=53.44635321212876,  lon=10.064849853515625 },
+        { lat=53.40595029739904,  lon=9.985198974609375  },
+        { lat=53.42385506057106,  lon=9.951210021972656  },
+        { lat=53.41843327091211,  lon=9.944171905517578  },
+        { lat=53.41812635648326,  lon=9.927349090576172  },
+        { lat=53.412294561442884, lon=9.917736053466797  },
+        { lat=53.41464783813818,  lon=9.901256561279297  },
+        { lat=53.443490472483326, lon=9.912586212158201  },
+        { lat=53.45177144115704,  lon=9.897651672363281  },
+        { lat=53.43633277935392,  lon=9.866924285888672  },
+        { lat=53.427639673754776, lon=9.866409301757812  },
+        { lat=53.427639673754776, lon=9.858856201171875  },
+        { lat=53.46710230573499,  lon=9.795513153076172  },
+        { lat=53.49039461941655,  lon=9.795341491699219  },
+        { lat=53.49029248806277,  lon=9.77903366088867   },
+        { lat=53.49856433088649,  lon=9.780235290527344  },
+        { lat=53.5078554643033,   lon=9.758434295654297  },
+        { lat=53.545407634092975, lon=9.759807586669922  },
+        { lat=53.568147234570084, lon=9.633293151855469  },
+        { lat=53.58802162343514,  lon=9.655780792236328  },
+        { lat=53.568351121879815, lon=9.727706909179688  },
+        { lat=53.60921067445695,  lon=9.737663269042969  },
+    }
+
+    points = {
+        {
+            name='Alice',
+            lat=52.52437,
+            lon=13.41053,
+        }, {
+            name='Bob',
+            lat=53.551085,
+            lon=9.993682,
+        }, {
+            name='Charlie',
+            lat=40.726662,
+            lon=-74.036677,
+        }
+    }
+
+    for i=1, #points do
+        if geo_polygon_contains(hamburg, points[i]) then
+            info('[INSIDE ] ' .. points[i]['name'])
+        else
+            info('[OUTSIDE] ' .. points[i]['name'])
+        end
+    end
+end

modules/harness/google-tls.lua

@@ -0,0 +1,29 @@
+-- Description: Test various tls functions
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('sending https request to google.com')
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://google.com/', {})
+    r = http_send(req)
+    if last_err() then return end
+    debug(r)
+
+    info('creating tls socket to google.com')
+    sock = sock_connect('google.com', 443, {
+        tls=true,
+    })
+    if last_err() then return end
+    debug(sock)
+
+    info('creating socket to google.com, wrapping afterwards')
+    sock = sock_connect('google.com', 443, {})
+    if last_err() then return end
+    tls = sock_upgrade_tls(sock, {
+        sni_value='google.com',
+    })
+    if last_err() then return end
+    debug(sock)
+    debug(tls)
+end

modules/harness/notify-ohai.lua

@@ -0,0 +1,8 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/notify-ratelimit.lua

@@ -0,0 +1,9 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    ratelimit_throttle('foo', 3, 10000)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/notify-slow.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    -- TODO your code here
+    sleep(1)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/pgp-fetch.lua

@@ -0,0 +1,22 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+    url = 'https://openpgpkey.archlinux.org/.well-known/openpgpkey/archlinux.org/hu/in9mwr4s84x7gm51851h343n3at1x61g?l=anthraxx'
+
+    req = http_request(session, 'GET', url, {})
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['text'])
+    info(k)
+
+    req = http_request(session, 'GET', url, {
+        binary=true,
+    })
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['binary'])
+    info(k)
+end

modules/harness/ratelimit.lua

@@ -0,0 +1,11 @@
+-- Description: Run script with a global ratelimit
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- this shouldn't complete in less than 5 seconds
+    for i=1, 20 do
+        ratelimit_throttle('foo', 4, 1000)
+        info(sn0int_time())
+    end
+end

modules/harness/socket-ping.lua

@@ -0,0 +1,33 @@
+-- Description: Connect somewhere and send a ping every 3s
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+INTERVAL = 3
+
+function run()
+    local sock = sock_connect('127.0.0.1', 4444, {
+        read_timeout=INTERVAL,
+    })
+    if last_err() then return end
+
+    local last_ping = time_unix()
+    while true do
+        local now = time_unix()
+        local sleep = last_ping + INTERVAL - now
+
+        if sleep <= 0 then
+            sock_send(sock, sn0int_time() .. ' ping\n')
+            last_ping = now
+            sleep = INTERVAL
+        end
+
+        sock_options(sock, {
+            read_timeout=sleep,
+        })
+        if last_err() then return end
+
+        local buf = sock_recv(sock)
+        if last_err() then return end
+        info(buf)
+    end
+end

modules/harness/url-inserter.lua

@@ -0,0 +1,58 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('preparing')
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value='example.com',
+    })
+
+    info('inserting')
+    url1 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com',
+    })
+    url2 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com/ohai',
+        body='ohai',
+    })
+    url3 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com/world',
+        body={0x77, 0x6f, 0x72, 0x6c, 0x64},
+    })
+
+    info('updating')
+    db_update('url', {
+        id=url1,
+        subdomain_id=subdomain_id,
+        value='https://example.com',
+        path='/',
+        unscoped=false,
+    }, {
+    })
+    db_update('url', {
+        id=url2,
+        subdomain_id=subdomain_id,
+        value='https://example.com/ohai',
+        path='/ohai',
+        body='ohai',
+        unscoped=false,
+    }, {
+    })
+    db_update('url', {
+        id=url3,
+        subdomain_id=subdomain_id,
+        value='https://example.com/world',
+        path='/world',
+        body={0x77, 0x6f, 0x72, 0x6c, 0x64},
+        unscoped=false,
+    }, {
+    })
+end

modules/harness/winkekatze-sub.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local sock = mqtt_connect('mqtt://mqtt.winkekatze24.de', {
+        read_timeout=10,
+    })
+    if last_err() then return end
+    mqtt_subscribe(sock, '#', 0)
+
+    while true do
+        local pkt = mqtt_recv_text(sock)
+        if last_err() then return end
+        info(pkt)
+    end
+end

modules/harness/ws-ping.lua

@@ -0,0 +1,33 @@
+-- Description: Connect somewhere and send a ping every 3s
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+INTERVAL = 3
+
+function run()
+    local sock = ws_connect('ws://127.0.0.1:8080', {
+        read_timeout=INTERVAL,
+    })
+    if last_err() then return end
+
+    local last_ping = time_unix()
+    while true do
+        local now = time_unix()
+        local sleep = last_ping + INTERVAL - now
+
+        if sleep <= 0 then
+            ws_send_text(sock, sn0int_time() .. ' ping\n')
+            last_ping = now
+            sleep = INTERVAL
+        end
+
+        ws_options(sock, {
+            read_timeout=sleep,
+        })
+        if last_err() then return end
+
+        local buf = ws_recv_text(sock)
+        if last_err() then return end
+        info(buf)
+    end
+end

modules/harness/ws.lua

@@ -0,0 +1,23 @@
+-- Description: Create an echo websocket connection
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local target = 'ws://echo.websocket.org'
+
+    info('connecting to ' .. target)
+    local sock = ws_connect(target, {})
+    if last_err() then return end
+
+    info('sending')
+    ws_send_text(sock, 'ohai wurld')
+    if last_err() then return end
+
+    info('recieving')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'ohai wurld' then
+        return 'echo failed, got: ' .. msg
+    end
+end

modules/harness/wss.lua

@@ -0,0 +1,30 @@
+-- Description: Create an encrypted websocket connection
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- local target = 'ws://echo.websocket.org' -- doesn't support proper ciphers
+    local target = 'wss://rocket.events.ccc.de/sockjs/258/whi0yr1y/websocket'
+
+    info('connecting to ' .. target)
+    local sock = ws_connect(target, {})
+    if last_err() then return end
+
+    info('recieving 1/2')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'o' then
+        return 'recieve failed, got ' .. msg
+    end
+
+    info('recieving 2/2')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'a["{\\"server_id\\":\\"0\\"}"]' then
+        return 'recieve failed, got ' .. msg
+    end
+
+    info('handshake succeeded')
+end

sn0int-common/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "sn0int-common"
-version = "0.9.0"
-description = "Common code for sn0int"
+version = "0.11.0"
+description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"

sn0int-common/src/metadata.rs

@@ -44,6 +44,7 @@ pub enum Source {
     Netblocks,
     CryptoAddrs(Option<String>),
     KeyRing(String),
+    Notifications,
 }
 
 impl Source {
@@ -63,6 +64,7 @@ impl Source {
             Source::Ports => "ports",
             Source::Netblocks => "netblocks",
             Source::CryptoAddrs(_) => "cryptoaddrs",
+            Source::Notifications => "notifications",
             Source::KeyRing(_) => "keyring",
         }
     }
@@ -94,6 +96,7 @@ impl FromStr for Source {
             ("ports", None) => Ok(Source::Ports),
             ("netblocks", None) => Ok(Source::Netblocks),
             ("cryptoaddrs", param) => Ok(Source::CryptoAddrs(param.map(String::from))),
+            ("notifications", None) => Ok(Source::Notifications),
             ("keyring", Some(param)) => Ok(Source::KeyRing(param.to_string())),
             (x, Some(param)) => bail!("Unknown Source: {:?} ({:?})", x, param),
             (x, None) => bail!("Unknown Source: {:?}", x),

sn0int-registry/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.9.0", path="sn0int-common" }
+sn0int-common = { version="0.11.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }

sn0int-registry/Dockerfile

@@ -2,16 +2,16 @@ FROM rust:buster
 RUN apt-get update -q && apt-get install -yq llvm libclang-dev \
     && rm -rf /var/lib/apt/lists/*
 RUN rustup install nightly
-WORKDIR /usr/src/sn0int-registry
+WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo +nightly build --release --verbose
+RUN cd sn0int-registry && cargo +nightly build --release --verbose
 RUN strip target/release/sn0int-registry
 
 FROM debian:buster
 RUN apt-get update -q && apt-get install -yq libcurl4 libpq5 \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=0 /usr/src/sn0int-registry/target/release/sn0int-registry /usr/local/bin/sn0int-registry
-COPY templates /templates
+COPY --from=0 /usr/src/sn0int/target/release/sn0int-registry /usr/local/bin/sn0int-registry
+COPY sn0int-registry/templates /templates
 ENV ROCKET_ENV=prod \
     ROCKET_ADDRESS=0.0.0.0 \
     ROCKET_PORT=8000

sn0int-registry/src/auth.rs

@@ -2,7 +2,7 @@ use diesel::pg::PgConnection;
 use oauth2::basic::BasicClient;
 use oauth2::prelude::*;
 use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl, TokenResponse};
-use crate::github::GithubAuthenticator;
+use crate::github;
 use sn0int_registry::errors::*;
 use sn0int_registry::models::AuthToken;
 use url::Url;
@@ -67,8 +67,7 @@ impl Authenticator {
         let access_token = response.access_token();
         let access_token = access_token.secret().to_string();
 
-        let client = GithubAuthenticator::from_env()?;
-        let user = client.get_username(&access_token)?;
+        let user = github::get_username(&access_token)?;
 
         AuthToken::create(&AuthToken {
             id: state,

sn0int-registry/src/auth2.rs

@@ -4,7 +4,7 @@ use sn0int_registry::db::Connection;
 use rocket::http::Status;
 use rocket::{Request, Outcome};
 use rocket::request::{self, FromRequest};
-use crate::github::GithubAuthenticator;
+use crate::github;
 
 
 pub struct AuthHeader(String);
@@ -12,8 +12,7 @@ pub struct AuthHeader(String);
 impl AuthHeader {
     pub fn verify(&self, connection: &Connection) -> Result<String> {
         let session = AuthToken::read(&self.0, &connection)?;
-        let client = GithubAuthenticator::from_env()?;
-        client.get_username(&session.access_token)
+        github::get_username(&session.access_token)
             .map_err(Error::from)
     }
 }

sn0int-registry/src/github.rs

@@ -1,54 +1,23 @@
 use sn0int_registry::errors::*;
-use std::env;
 use reqwest;
 
+pub fn get_username(oauth_token: &str) -> Result<String> {
+    let client = reqwest::Client::new();
+    let mut resp = client.get("https://api.github.com/user")
+        .header("Authorization", format!("token {}", oauth_token))
+        .send()
+        .context("Failed to check access_token")?
+        .error_for_status()
+        .context("Github returned http error")?;
 
-pub struct GithubAuthenticator {
-    client_id: String,
-    client_secret: String,
-}
+    let data = resp.json::<GithubUser>()
+        .context("Failed to deserialize github reply")?;
 
-impl GithubAuthenticator {
-    pub fn new(client_id: String, client_secret: String) -> GithubAuthenticator {
-        GithubAuthenticator {
-            client_id,
-            client_secret,
-        }
-    }
-
-    pub fn from_env() -> Result<GithubAuthenticator> {
-        let client_id = env::var("GITHUB_CLIENT_ID")
-            .context("GITHUB_CLIENT_ID is not set")?;
-        let client_secret = env::var("GITHUB_CLIENT_SECRET")
-            .context("GITHUB_CLIENT_SECRET is not set")?;
-        Ok(GithubAuthenticator::new(client_id, client_secret))
-    }
-
-    pub fn get_username(&self, oauth_token: &str) -> Result<String> {
-        let url = format!("https://api.github.com/applications/{}/tokens/{}", self.client_id, oauth_token);
-        let client = reqwest::Client::new();
-        let mut resp = client.get(&url)
-            .basic_auth(&self.client_id, Some(&self.client_secret))
-            .send()?;
-
-        if !resp.status().is_success() {
-            bail!("Github returned: {}", resp.status())
-        }
-
-        let data = resp.json::<GithubReply>()
-            .context("Failed to deserialize github reply")?;
-
-        Ok(data.user.login)
-    }
+    Ok(data.login)
 }
 
 #[derive(Debug, Deserialize)]
-pub struct GithubReply {
-    user: GithubUser,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct GithubUser {
+struct GithubUser {
     login: String,
     #[serde(rename="type")]
     user_type: String,

sn0int-registry/templates/index.html.hbs

@@ -21,7 +21,7 @@
     To install a module run:
     </p>
     <p class="code"><code>
-    sn0int install kpcyrd/ctlogs
+    sn0int pkg install kpcyrd/ctlogs
     </code></p>
 
     {{#each modules}}

sn0int-std/Cargo.toml

@@ -0,0 +1,56 @@
+[package]
+name = "sn0int-std"
+version = "0.19.0"
+description = "sn0int - stdlib"
+authors = ["kpcyrd <git@rxv.cc>"]
+repository = "https://github.com/kpcyrd/sn0int"
+license = "GPL-3.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+log = "0.4"
+failure = "0.1"
+hlua-badtouch = "0.4"
+tokio = "0.1.14"
+rand = "0.7"
+regex = "1.0"
+rustls = { version="0.16", features=["dangerous_configuration"] }
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+serde_urlencoded = "0.6"
+webpki = "0.21"
+webpki-roots = "0.19.0"
+ct-logs = "0.6"
+#chrootable-https = { path = "../../chrootable-https" }
+chrootable-https = "0.15.2"
+http = "0.2"
+bufstream = "0.1.4"
+pem = "0.7"
+url = "2.0"
+tungstenite = { version = "0.10.1", default-features = false }
+kuchiki = "0.8.0"
+maxminddb = "0.14"
+x509-parser = "0.7"
+der-parser = "3.0"
+publicsuffix = { version="1.5", default-features=false }
+xml-rs = "0.8"
+geo = "0.13"
+bytes = "0.4"
+base64 = "0.12"
+chrono = { version = "0.4", features = ["serde"] }
+mqtt-protocol = "0.8.1"
+sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }
+
+image = "0.23.0"
+kamadak-exif = "0.5.1"
+
+bs58 = "0.3"
+digest = "0.8.0"
+blake2 = "0.8.0"
+
+[dev-dependencies]
+env_logger = "0.7"
+maplit = "1.0.1"

sn0int-std/src/blobs.rs

@@ -0,0 +1,118 @@
+use bytes::Bytes;
+use blake2::VarBlake2b;
+use digest::{Input, VariableOutput};
+use serde::ser::{Serialize, Serializer};
+use serde::de::{self, Deserialize, Deserializer};
+use std::result;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Blob {
+    pub id: String,
+    pub bytes: Bytes,
+}
+
+impl Blob {
+    pub fn create(bytes: Bytes) -> Blob {
+        let id = Self::hash(&bytes);
+        Blob {
+            id,
+            bytes,
+        }
+    }
+
+    pub fn hash(bytes: &[u8]) -> String {
+        let mut h = VarBlake2b::new(32).unwrap();
+        h.input(bytes);
+        Self::encode_hash(&h.vec_result())
+    }
+
+    #[inline]
+    fn encode_hash(bytes: &[u8]) -> String {
+        let x = bs58::encode(bytes).into_string();
+        format!("{:0<44}", x)
+    }
+}
+
+impl Serialize for Blob {
+    #[inline]
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let s = base64::encode(&self.bytes);
+        serializer.serialize_str(&s)
+    }
+}
+
+impl<'de> Deserialize<'de> for Blob {
+    #[inline]
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let bytes = base64::decode(&s)
+            .map_err(de::Error::custom)?;
+        Ok(Blob::create(Bytes::from(bytes)))
+    }
+}
+
+pub trait BlobState {
+    fn register_blob(&self, blob: Blob) -> String;
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json;
+
+    #[inline]
+    fn blob() -> (Bytes, Blob) {
+        let bytes = Bytes::from(&b"asdf"[..]);
+        (bytes.clone(), Blob::create(bytes))
+    }
+
+    #[test]
+    fn verify_create_blob() {
+        let (bytes, blob) = blob();
+        assert_eq!(blob, Blob {
+            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
+            bytes,
+        });
+    }
+
+    #[test]
+    fn test_serialize() {
+        let (_, blob) = blob();
+        let json = serde_json::to_string(&blob).unwrap();
+        assert_eq!(&json, "\"YXNkZg==\"");
+    }
+
+    #[test]
+    fn test_deserialize() {
+        let (_, blob1) = blob();
+        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
+        assert_eq!(blob1, blob2);
+    }
+
+    #[test]
+    fn test_hash_encoding() {
+        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
+    }
+
+    #[test]
+    fn test_hash_encoding_padding() {
+        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
+    }
+}

sn0int-std/src/crt.rs

@@ -11,6 +11,7 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 #[derive(Debug, PartialEq)]
 pub enum AlternativeName {
     DnsName(String),
+    Email(String),
     IpAddr(IpAddr),
 }
 
@@ -37,6 +38,7 @@ pub fn san_value(o: DerObject) -> Result<AlternativeName> {
 
     match (o.class, o.tag, &o.content) {
         (2, BerTag::Integer, BerObjectContent::Unknown(BerTag::Integer, value)) => san_value_dns(value),
+        (2, BerTag::Boolean, BerObjectContent::Unknown(BerTag::Boolean, value)) => san_value_email(value),
         (2, BerTag::ObjDescriptor, BerObjectContent::Unknown(BerTag::ObjDescriptor, value)) => san_value_ipaddr(value),
         _ => bail!("Unexpected object: {:?}", o),
     }
@@ -49,6 +51,13 @@ pub fn san_value_dns(v: &[u8]) -> Result<AlternativeName> {
         .map_err(Error::from)
 }
 
+pub fn san_value_email(v: &[u8]) -> Result<AlternativeName> {
+    debug!("Reading as email: {:?}", v);
+    String::from_utf8(v.to_vec())
+        .map(AlternativeName::Email)
+        .map_err(Error::from)
+}
+
 pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
     debug!("Reading as ipaddr: {:?}", v);
     match v.len() {
@@ -68,7 +77,8 @@ pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
     pub valid_names: Vec<String>,
-    pub valid_ipaddrs: Vec<IpAddr>
+    pub valid_emails: Vec<String>,
+    pub valid_ipaddrs: Vec<IpAddr>,
 }
 
 impl Certificate {
@@ -103,6 +113,7 @@ impl Certificate {
         };
 
         let mut valid_names = HashSet::new();
+        let mut valid_emails = HashSet::new();
         let mut valid_ipaddrs = HashSet::new();
 
         for x in crt.tbs_certificate.subject.rdn_seq {
@@ -131,15 +142,18 @@ impl Certificate {
             for v in values {
                 match v {
                     AlternativeName::DnsName(v) => valid_names.insert(v),
+                    AlternativeName::Email(v) => valid_emails.insert(v),
                     AlternativeName::IpAddr(v) => valid_ipaddrs.insert(v),
                 };
             }
         }
 
         let valid_names = valid_names.into_iter().collect();
+        let valid_emails = valid_emails.into_iter().collect();
         let valid_ipaddrs = valid_ipaddrs.into_iter().collect();
         Ok(Certificate {
             valid_names,
+            valid_emails,
             valid_ipaddrs,
         })
     }
@@ -197,6 +211,7 @@ WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik=
         x.valid_names.sort();
         assert_eq!(x, Certificate {
             valid_names: vec!["github.com".into(), "www.github.com".into()],
+            valid_emails: vec![],
             valid_ipaddrs: vec![],
         });
     }
@@ -235,6 +250,7 @@ ZkZZmqNn2Q8=
                 "*.cloudflare-dns.com".into(),
                 "cloudflare-dns.com".into(),
             ],
+            valid_emails: vec![],
             valid_ipaddrs: vec![
                 "1.0.0.1".parse().unwrap(),
                 "1.1.1.1".parse().unwrap(),
@@ -351,6 +367,7 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
                 "witt-weiden.dam.staging.aboutyou.cloud".into(),
                 "www.aboutyou.de".into(),
             ],
+            valid_emails: vec![],
             valid_ipaddrs: vec![],
         });
     }
@@ -396,4 +413,50 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
             .expect("Failed to process san value");
         assert_eq!(v, AlternativeName::IpAddr("1.1.1.1".parse().unwrap()));
     }
+
+    #[test]
+    fn test_san_email() {
+        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+MIIE5zCCA8+gAwIBAgIQBvsKfZ5AGSW3Vc8Ldto1hTANBgkqhkiG9w0BAQUFADBp
+MSQwIgYJKoZIhvcNAQkBFhVwa2lfYWRtaW5Ac3VuZ2FyZC5jb20xJjAkBgNVBAoT
+HVN1bkdhcmQgQXZhaWxhYmlsaXR5IFNlcnZpY2VzMRkwFwYDVQQDExBTQVMgUHVi
+bGljIENBIHYxMB4XDTEwMDkwMjE2MzY0OVoXDTExMTAwMTE2MzEwMFowgbQxCzAJ
+BgNVBAYTAlVTMQswCQYDVQQIEwJPUjERMA8GA1UEBxMIUG9ydGxhbmQxEzARBgoJ
+kiaJk/IsZAEZFgNjb20xHDAaBgoJkiaJk/IsZAEZFgxqaXZlc29mdHdhcmUxHDAa
+BgNVBAoTE0ppdmUgU29mdHdhcmUsIEluYy4xEDAOBgNVBAsTB0hvc3RpbmcxIjAg
+BgNVBAMTGSouaG9zdGVkLmppdmVzb2Z0d2FyZS5jb20wggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQC0oornTIyL5YjZMpNwy+V2YJbLqaLrbPrbWFCsNJDx
+dnubjfR71aW+YYlUZF8zoq4jFetkblCehyvPEb5tD/l3/WZhiXYOziPDrsEVCngF
+3/b0H3Dyk6mNWBZcNpJkdpOx1YB6Zer8eKzFOr7Qj3aevOR/bEe2NARJIaO0Rjwe
+YIWY0arKRm6z4nJD8fYAvFV6wRWmHsZO9ci7hiGeW3YL6jQYJqLeuwXm64l0jptb
+Qg8r8c1V5BXETlvQJL34gUozEl9jDpzR7KoXtErhlU2ytl9Wg+fOxYuWgx8vER0/
+7Hqc/qD5e7B+NtwgfEio7SvNGA/HhjNxW2Wbrx4qooJRAgMBAAGjggE9MIIBOTAO
+BgNVHQ8BAf8EBAMCA6gwEQYJYIZIAYb4QgEBBAQDAgbAMCIGA1UdEQQbMBmBF3N1
+YmplY3RuYW1lQGV4YW1wbGUuY29tMB8GA1UdIwQYMBaAFDhBxvKFgYP96+IaNpI7
+JmEWgRESMFQGA1UdIARNMEswSQYJKoZIhvcNBQYBMDwwOgYIKwYBBQUHAgEWLmh0
+dHBzOi8vY2VydGlmaWNhdGUuc3VuZ2FyZC5jb20vU0FTX0NBX0NQUy5wZGYwRQYD
+VR0fBD4wPDA6oDigNoY0aHR0cHM6Ly9jZXJ0aWZpY2F0ZS5zdW5nYXJkLmNvbS9T
+QVNfUHVibGljX0NBX3YxLmNybDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4E
+FgQUhJ99py6oeCYBzcePPjhOxHVDBvwwDQYJKoZIhvcNAQEFBQADggEBAF/DAJgX
+f50x8t8Im96AUn4DqC+T0QZIYHihpj2uCwWDbdp5efppqTrk6FrpFOzQy0TRkstb
+Q3zKSgduedQiwii9qh88O1h2gbSTqfi55ApOIGoCiiRCqio2p4tbKyqPV3Q0eyYw
+K4f9GAOcawvNsI//mx99ol/ZGEamydeL9G0qiKrhqSxd2TGmFaIVJdu9fh59hos4
+6t9c4FVyYygdsIeGHkHjpB2bKjZhnJpKRh9dGWctcjdHMITBBqgiRH9OZa/w6SPE
+UT+11L6q7MSIXSIMV8kJSUUYE92P7bnAqViTIuu/hHnfmIhiy6t7AuT2QHEhqDab
+EF4l5MwdUqs8FvM=
+-----END CERTIFICATE-----
+"#).expect("Failed to parse cert");
+        x.valid_names.sort();
+        x.valid_emails.sort();
+        x.valid_ipaddrs.sort();
+        assert_eq!(x, Certificate {
+            valid_names: vec![
+                "*.hosted.jivesoftware.com".into(),
+            ],
+            valid_emails: vec![
+                "subjectname@example.com".into(),
+            ],
+            valid_ipaddrs: vec![],
+        });
+    }
 }

sn0int-std/src/crypto.rs

@@ -0,0 +1,51 @@
+use crate::errors::*;
+use sodiumoxide::crypto::secretbox::{self, Key, Nonce};
+use std::iter;
+
+pub fn key_trunc_pad(mut key: &[u8], len: usize, pad: u8) -> Vec<u8> {
+    if key.len() > len {
+        key = &key[..len];
+    }
+
+    let mut key = key.to_vec();
+    key.extend(iter::repeat(pad).take(len - key.len()));
+    key
+}
+
+pub fn sodium_secretbox_open(encrypted: &[u8], key: &[u8]) -> Result<Vec<u8>> {
+    if encrypted.len() <= secretbox::NONCEBYTES {
+        bail!("Encrypted message is too short");
+    }
+
+    let key = Key::from_slice(key)
+        .ok_or_else(|| format_err!("Key has wrong length"))?;
+    let nonce = Nonce::from_slice(&encrypted[..secretbox::NONCEBYTES])
+        .ok_or_else(|| format_err!("Nonce has wrong length"))?;
+    let ciphertext = &encrypted[secretbox::NONCEBYTES..];
+    let plain = secretbox::open(&ciphertext, &nonce, &key)
+        .map_err(|_| format_err!("Failed to decrypt secretbox"))?;
+    Ok(plain)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_key_equal() {
+        let key = key_trunc_pad(&[1, 2, 3, 4, 5], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn test_key_trunc() {
+        let key = key_trunc_pad(&[1, 2, 3, 4, 5, 6, 7, 8, 9], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn test_key_pad() {
+        let key = key_trunc_pad(&[1, 2, 3], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 0, 0]);
+    }
+}

sn0int-std/src/engine/mod.rs

@@ -0,0 +1 @@
+pub mod structs;

sn0int-std/src/engine/structs.rs

@@ -41,6 +41,12 @@ impl LuaMap {
     pub fn insert_num<K: Into<String>>(&mut self, k: K, v: f64) {
         self.0.insert(AnyHashableLuaValue::LuaString(k.into()), AnyLuaValue::LuaNumber(v));
     }
+
+    pub fn insert_serde<K: Into<String>, S: serde::Serialize>(&mut self, k: K, v: S) -> Result<()> {
+        let v = serde_json::to_value(v)?;
+        self.0.insert(AnyHashableLuaValue::LuaString(k.into()), LuaJsonValue::from(v).into());
+        Ok(())
+    }
 }
 
 impl IntoIterator for LuaMap {

sn0int-std/src/errors.rs

@@ -0,0 +1,3 @@
+pub use log::{trace, debug, info, warn, error};
+pub use failure::{Error, ResultExt, format_err, bail};
+pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-std/src/geo.rs

@@ -0,0 +1,111 @@
+use crate::errors::*;
+
+use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
+use geo::{LineString, Polygon, Coordinate};
+use geo::prelude::*;
+
+
+#[derive(Debug, Deserialize)]
+pub struct Point {
+    lon: f64,
+    lat: f64,
+}
+
+impl Point {
+    pub fn try_from(x: AnyLuaValue) -> Result<Point> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub fn polygon_contains(ring: &[Point], p: &Point) -> bool {
+    let ring = ring.iter()
+        .map(|p| Coordinate { x: p.lon, y: p.lat })
+        .collect::<Vec<_>>();
+
+    let polygon = Polygon::new(LineString::from(ring), vec![]);
+    let point = geo::Point::new(p.lon, p.lat);
+
+    polygon.contains(&point)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn hamburg_polygon() -> &'static [Point] {
+        &[
+            Point { lat: 53.63975308945899,  lon: 9.764785766601562  },
+            Point { lat: 53.59494998253459,  lon: 9.827270507812     },
+            Point { lat: 53.663153974456456, lon: 9.9151611328125    },
+            Point { lat: 53.65582987649682,  lon: 9.976272583007812  },
+            Point { lat: 53.68613523817129,  lon: 9.992752075195312  },
+            Point { lat: 53.68674518938816,  lon: 10.051460266113281 },
+            Point { lat: 53.72495117617815,  lon: 10.075492858886719 },
+            Point { lat: 53.71946627930625,  lon: 10.118408203125    },
+            Point { lat: 53.743635083157756, lon: 10.164413452148438 },
+            Point { lat: 53.73104466704585,  lon: 10.202865600585938 },
+            Point { lat: 53.676781546441546, lon: 10.16304016113281  },
+            Point { lat: 53.632832079199474, lon: 10.235824584960938 },
+            Point { lat: 53.608803292930894, lon: 10.2008056640625   },
+            Point { lat: 53.578646152866504, lon: 10.208358764648438 },
+            Point { lat: 53.57212285981298,  lon: 10.163726806640625 },
+            Point { lat: 53.52071674896369,  lon: 10.18707275390625  },
+            Point { lat: 53.52643162253097,  lon: 10.224151611328125 },
+            Point { lat: 53.44062753992289,  lon: 10.347747802734375 },
+            Point { lat: 53.38824275010831,  lon: 10.248870849609375 },
+            Point { lat: 53.38824275010831,  lon: 10.15960693359375  },
+            Point { lat: 53.44635321212876,  lon: 10.064849853515625 },
+            Point { lat: 53.40595029739904,  lon: 9.985198974609375  },
+            Point { lat: 53.42385506057106,  lon: 9.951210021972656  },
+            Point { lat: 53.41843327091211,  lon: 9.944171905517578  },
+            Point { lat: 53.41812635648326,  lon: 9.927349090576172  },
+            Point { lat: 53.412294561442884, lon: 9.917736053466797  },
+            Point { lat: 53.41464783813818,  lon: 9.901256561279297  },
+            Point { lat: 53.443490472483326, lon: 9.912586212158201  },
+            Point { lat: 53.45177144115704,  lon: 9.897651672363281  },
+            Point { lat: 53.43633277935392,  lon: 9.866924285888672  },
+            Point { lat: 53.427639673754776, lon: 9.866409301757812  },
+            Point { lat: 53.427639673754776, lon: 9.858856201171875  },
+            Point { lat: 53.46710230573499,  lon: 9.795513153076172  },
+            Point { lat: 53.49039461941655,  lon: 9.795341491699219  },
+            Point { lat: 53.49029248806277,  lon: 9.77903366088867   },
+            Point { lat: 53.49856433088649,  lon: 9.780235290527344  },
+            Point { lat: 53.5078554643033,   lon: 9.758434295654297  },
+            Point { lat: 53.545407634092975, lon: 9.759807586669922  },
+            Point { lat: 53.568147234570084, lon: 9.633293151855469  },
+            Point { lat: 53.58802162343514,  lon: 9.655780792236328  },
+            Point { lat: 53.568351121879815, lon: 9.727706909179688  },
+            Point { lat: 53.60921067445695,  lon: 9.737663269042969  },
+        ]
+    }
+
+    #[test]
+    fn test_polygon_hamburg_contains_hamburg() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 53.551085,
+            lon: 9.993682,
+        });
+        assert!(contains);
+    }
+
+    #[test]
+    fn test_polygon_hamburg_not_contains_berlin() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 52.52437,
+            lon: 13.41053,
+        });
+        assert!(!contains);
+    }
+
+    #[test]
+    fn test_polygon_hamburg_not_contains_ny() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 40.726662,
+            lon: -74.036677,
+        });
+        assert!(!contains);
+    }
+}

sn0int-std/src/geoip/mod.rs

@@ -1,57 +1,45 @@
-use chrootable_https::Client;
-use crate::archive;
 use crate::errors::*;
 use crate::lazy::LazyInit;
-use crate::paths;
-use crate::worker;
 use maxminddb::{self, geoip2};
 use std::fmt;
 use std::fs::{self, File};
 use std::net::IpAddr;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::io::Read;
 use std::sync::Arc;
 
-pub static GEOIP_CITY_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz";
-pub static GEOIP_ASN_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz";
-
 pub mod models;
 use self::models::GeoLookup;
 use self::models::AsnLookup;
 
 
 pub trait Maxmind: Sized {
-    fn archive_filename() -> &'static str;
-
-    fn archive_url() -> &'static str;
+    fn filename() -> &'static str;
 
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self;
 
-    // TODO: refactor this to return Path
-    fn cache_path() -> Result<String> {
+    fn cache_path(cache_dir: &Path) -> Result<PathBuf> {
         // use system path if exists
         for path in &[
             // Archlinux
             "/usr/share/GeoIP/",
             // OpenBSD
             "/usr/local/share/examples/libmaxminddb/",
+            // geoipupdate
+            "/var/lib/GeoIP/",
         ] {
             let path = Path::new(path);
-            let path = path.join(Self::archive_filename());
+            let path = path.join(Self::filename());
 
             if path.exists() {
-                let path = path.to_str()
-                    .ok_or_else(|| format_err!("Failed to decode path"))?;
-                return Ok(path.to_string());
+                return Ok(path);
             }
         }
 
         // use cache path
-        let path = paths::cache_dir()?
-            .join(Self::archive_filename());
-        let path = path.to_str()
-            .ok_or_else(|| format_err!("Failed to decode path"))?;
-        Ok(path.to_string())
+        let path = cache_dir
+            .join(Self::filename());
+        Ok(path)
     }
 
     fn from_buf(buf: Vec<u8>) -> Result<Self> {
@@ -60,37 +48,20 @@ pub trait Maxmind: Sized {
         Ok(Self::new(reader))
     }
 
-    fn open(path: &str) -> Result<Self> {
+    fn open(path: &Path) -> Result<Self> {
         let buf = fs::read(path)?;
         Self::from_buf(buf)
     }
 
-    fn open_reader() -> Result<MaxmindReader> {
-        let path = Self::cache_path()?;
-        MaxmindReader::open_path(path)
-    }
+    fn try_open_reader(cache_dir: &Path) -> Result<Option<MaxmindReader>> {
+        let path = Self::cache_path(cache_dir)?;
 
-    fn open_or_download() -> Result<Self> {
-        let path = Self::cache_path()?;
-
-        if File::open(&path).is_err() {
-            worker::spawn_fn(&format!("Downloading {:?}", Self::archive_filename()), || {
-                Self::download(&path, Self::archive_filename(), Self::archive_url())
-            }, false)?;
-        };
-
-        Self::open(&path)
-    }
-
-    fn download<P: AsRef<Path>>(path: P, filter: &str, url: &str) -> Result<()> {
-        debug!("Downloading {:?}...", url);
-        let client = Client::with_system_resolver()?;
-        let resp = client.get(url)
-            .wait_for_response()
-            .context("http request failed")?;
-        debug!("Downloaded {} bytes", resp.body.len());
-        archive::extract(&mut &resp.body[..], filter, path)?;
-        Ok(())
+        if path.exists() {
+            let db = MaxmindReader::open_path(path)?;
+            Ok(Some(db))
+        } else {
+            Ok(None)
+        }
     }
 }
 
@@ -141,15 +112,10 @@ impl fmt::Debug for GeoIP {
 
 impl Maxmind for GeoIP {
     #[inline]
-    fn archive_filename() -> &'static str {
+    fn filename() -> &'static str {
         "GeoLite2-City.mmdb"
     }
 
-    #[inline]
-    fn archive_url() -> &'static str {
-        GEOIP_CITY_URL
-    }
-
     #[inline]
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
         GeoIP {
@@ -178,15 +144,10 @@ impl fmt::Debug for AsnDB {
 
 impl Maxmind for AsnDB {
     #[inline]
-    fn archive_filename() -> &'static str {
+    fn filename() -> &'static str {
         "GeoLite2-ASN.mmdb"
     }
 
-    #[inline]
-    fn archive_url() -> &'static str {
-        GEOIP_ASN_URL
-    }
-
     #[inline]
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
         AsnDB {
@@ -205,22 +166,29 @@ impl AsnDB {
 
 #[cfg(test)]
 mod tests {
+    // You need geoip setup on your system to run this
+    /*
     use super::*;
 
     #[test]
+    #[ignore]
     fn test_geoip_lookup() {
         let ip = "1.1.1.1".parse().unwrap();
-        let geoip = GeoIP::open_or_download().expect("Failed to load geoip");
+        let path = GeoIP::cache_path().unwrap();
+        let geoip = GeoIP::open(&path).unwrap();
         let lookup = geoip.lookup(ip).expect("GeoIP lookup failed");
         println!("{:#?}", lookup);
         assert_eq!(lookup.city, None);
     }
 
     #[test]
+    #[ignore]
     fn test_asn_lookup() {
         let ip = "1.1.1.1".parse().unwrap();
-        let asndb = AsnDB::open_or_download().expect("Failed to load asndb");
+        let path = AsnDB::cache_path().unwrap();
+        let asndb = AsnDB::open(&path).unwrap();
         let lookup = asndb.lookup(ip).expect("ASN lookup failed");
         println!("{:#?}", lookup);
     }
+    */
 }

sn0int-std/src/geoip/models.rs

@@ -3,14 +3,8 @@ use maxminddb::geoip2;
 use std::collections::BTreeMap;
 
 
-fn from_geoip_model_names(names: Option<BTreeMap<String, String>>) -> Option<String> {
-    let names = match names {
-        Some(names) => names,
-        _ => return None,
-    };
-
-    names.get("en")
-        .map(|x| x.to_owned())
+fn from_geoip_model_names(names: Option<BTreeMap<&str, &str>>) -> Option<String> {
+    names?.get("en").map(|x| x.to_string())
 }
 
 #[derive(Debug, Serialize)]
@@ -24,7 +18,7 @@ pub struct GeoLookup {
     pub longitude: Option<f64>,
 }
 
-impl From<geoip2::City> for GeoLookup {
+impl<'a> From<geoip2::City<'a>> for GeoLookup {
     fn from(lookup: geoip2::City) -> GeoLookup {
         // parse maxminddb lookup
         let continent = match lookup.continent {
@@ -79,17 +73,11 @@ pub struct Continent {
 
 impl Continent {
     fn from_maxmind(continent: geoip2::model::Continent) -> Option<Self> {
-        let code = match continent.code {
-            Some(code) => code,
-            _ => return None,
-        };
-        let name = match from_geoip_model_names(continent.names) {
-            Some(name) => name,
-            _ => return None,
-        };
+        let code = continent.code?;
+        let name = from_geoip_model_names(continent.names)?;
 
         Some(Continent {
-            code,
+            code: code.to_string(),
             name,
         })
     }
@@ -103,17 +91,11 @@ pub struct Country {
 
 impl Country {
     fn from_maxmind(country: geoip2::model::Country) -> Option<Self> {
-        let code = match country.iso_code {
-            Some(code) => code,
-            _ => return None,
-        };
-        let name = match from_geoip_model_names(country.names) {
-            Some(name) => name,
-            _ => return None,
-        };
+        let code = country.iso_code?;
+        let name = from_geoip_model_names(country.names)?;
 
         Some(Country {
-            code,
+            code: code.to_string(),
             name,
         })
     }
@@ -152,19 +134,14 @@ pub struct AsnLookup {
 impl AsnLookup {
     pub fn try_from(lookup: geoip2::Isp) -> Result<AsnLookup> {
         // parse maxminddb lookup
-        let asn = match lookup.autonomous_system_number {
-            Some(asn) => asn,
-            _ => bail!("autonomous_system_number not set"),
-        };
-        let as_org = match lookup.autonomous_system_organization {
-            Some(org) => org,
-            _ => bail!("autonomous_system_organization not set"),
-        };
+        let asn = lookup.autonomous_system_number
+            .ok_or_else(|| format_err!("autonomous_system_number not set"))?;
+        let as_org = lookup.autonomous_system_organization
+            .ok_or_else(|| format_err!("autonomous_system_organization not set"))?;
 
-        // return result
         Ok(AsnLookup {
             asn,
-            as_org,
+            as_org: as_org.to_string(),
         })
     }
 }

sn0int-std/src/gfx/exif.rs

@@ -11,11 +11,12 @@ pub struct Location {
 }
 
 impl Location {
-    pub fn try_from(fields: &[exif::Field]) -> Result<Location> {
+    fn try_from_iter<'a, I: IntoIterator<Item=&'a exif::Field>>(iter: I) -> Result<Self> {
         let mut builder = LocationBuilder::default();
-        fields.iter()
-            .map(|f| builder.add_one(f))
-            .collect::<Result<()>>()?;
+        for f in iter {
+            debug!("Exif field: {:?}", f.display_value().to_string());
+            builder.add_one(f)?;
+        }
         builder.build()
     }
 }
@@ -60,12 +61,12 @@ impl LocationBuilder {
 }
 
 pub fn gps(img: &[u8]) -> Result<Option<Location>> {
-    let mut buf = io::BufReader::new(img);
-    let reader = exif::Reader::new(&mut buf)?;
+    let mut buf = io::Cursor::new(img);
+    let reader = exif::Reader::new()
+        .read_from_container(&mut buf)?;
     let fields = reader.fields();
-    debug!("Exif fields: {:?}", fields);
 
-    let location = Location::try_from(fields).ok();
+    let location = Location::try_from_iter(fields).ok();
     Ok(location)
 }
 
@@ -115,18 +116,18 @@ mod tests {
     fn verify_exif_location() {
         test_init();
 
-        let location = Location::try_from(&[
+        let location = Location::try_from_iter(&[
             exif::Field {
                 tag: exif::Tag::GPSLatitudeRef,
-                thumbnail: false,
-                value: exif::Value::Ascii(vec![&[b'N']]),
+                ifd_num: exif::In::PRIMARY,
+                value: exif::Value::Ascii(vec![vec![b'N']]),
             }, exif::Field {
                 tag: exif::Tag::GPSLongitudeRef,
-                thumbnail: false,
-                value: exif::Value::Ascii(vec![&[b'E']]),
+                ifd_num: exif::In::PRIMARY,
+                value: exif::Value::Ascii(vec![vec![b'E']]),
             }, exif::Field {
                 tag: exif::Tag::GPSLatitude,
-                thumbnail: false,
+                ifd_num: exif::In::PRIMARY,
                 value: exif::Value::Rational(vec![exif::Rational {
                     num: 43,
                     denom: 1,
@@ -139,7 +140,7 @@ mod tests {
                 }]),
             }, exif::Field {
                 tag: exif::Tag::GPSLongitude,
-                thumbnail: false,
+                ifd_num: exif::In::PRIMARY,
                 value: exif::Value::Rational(vec![exif::Rational {
                     num: 11,
                     denom: 1,

sn0int-std/src/gfx/mod.rs

@@ -6,13 +6,13 @@ pub mod exif;
 
 #[derive(Debug)]
 pub enum ImageFormat {
-    PNG,
-    JPEG,
-    GIF,
-    WEBP,
-    TIFF,
-    BMP,
-    ICO,
+    Png,
+    Jpeg,
+    Gif,
+    WebP,
+    Tiff,
+    Bmp,
+    Ico,
 }
 
 impl ImageFormat {
@@ -20,28 +20,27 @@ impl ImageFormat {
         // https://www.iana.org/assignments/media-types/media-types.xhtml#image
         // /etc/nginx/mime.types
         match self {
-            ImageFormat::PNG => "image/png",
-            ImageFormat::JPEG => "image/jpeg",
-            ImageFormat::GIF => "image/gif",
-            ImageFormat::WEBP => "image/webp",
-            ImageFormat::TIFF => "image/tiff",
-            ImageFormat::BMP => "image/bmp",
-            ImageFormat::ICO => "image/vnd.microsoft.icon",
+            ImageFormat::Png => "image/png",
+            ImageFormat::Jpeg => "image/jpeg",
+            ImageFormat::Gif => "image/gif",
+            ImageFormat::WebP => "image/webp",
+            ImageFormat::Tiff => "image/tiff",
+            ImageFormat::Bmp => "image/bmp",
+            ImageFormat::Ico => "image/vnd.microsoft.icon",
         }
     }
 
     pub fn try_from(format: &image::ImageFormat) -> Result<ImageFormat> {
         use image::ImageFormat::*;
         match format {
-            PNG => Ok(ImageFormat::PNG),
-            JPEG => Ok(ImageFormat::JPEG),
-            GIF => Ok(ImageFormat::GIF),
-            WEBP => Ok(ImageFormat::WEBP),
-            TIFF => Ok(ImageFormat::TIFF),
-            BMP => Ok(ImageFormat::BMP),
-            ICO => Ok(ImageFormat::ICO),
-
-            HDR | PNM | TGA => bail!("Unsupported format: {:?}", format),
+            Png => Ok(ImageFormat::Png),
+            Jpeg => Ok(ImageFormat::Jpeg),
+            Gif => Ok(ImageFormat::Gif),
+            WebP => Ok(ImageFormat::WebP),
+            Tiff => Ok(ImageFormat::Tiff),
+            Bmp => Ok(ImageFormat::Bmp),
+            Ico => Ok(ImageFormat::Ico),
+            _ => bail!("Unsupported format: {:?}", format),
         }
     }
 }
@@ -49,13 +48,13 @@ impl ImageFormat {
 impl Into<image::ImageFormat> for ImageFormat {
     fn into(self) -> image::ImageFormat {
         match self {
-            ImageFormat::PNG => image::ImageFormat::PNG,
-            ImageFormat::JPEG => image::ImageFormat::JPEG,
-            ImageFormat::GIF => image::ImageFormat::GIF,
-            ImageFormat::WEBP => image::ImageFormat::WEBP,
-            ImageFormat::TIFF => image::ImageFormat::TIFF,
-            ImageFormat::BMP => image::ImageFormat::BMP,
-            ImageFormat::ICO => image::ImageFormat::ICO,
+            ImageFormat::Png => image::ImageFormat::Png,
+            ImageFormat::Jpeg => image::ImageFormat::Jpeg,
+            ImageFormat::Gif => image::ImageFormat::Gif,
+            ImageFormat::WebP => image::ImageFormat::WebP,
+            ImageFormat::Tiff => image::ImageFormat::Tiff,
+            ImageFormat::Bmp => image::ImageFormat::Bmp,
+            ImageFormat::Ico => image::ImageFormat::Ico,
         }
     }
 }
@@ -114,7 +113,7 @@ mod tests {
 
     #[test]
     fn verify_gfx_load_ico() {
-        let ico = fs::read("./sn0int-registry/assets/favicon.ico").expect("fs::read");
+        let ico = fs::read("../sn0int-registry/assets/favicon.ico").expect("fs::read");
         let img = load(&ico).expect("gfx::load");
 
         assert_eq!("image/vnd.microsoft.icon", img.mime());

sn0int-std/src/html.rs

@@ -1,4 +1,4 @@
-use crate::errors::Result;
+use crate::errors::*;
 
 use kuchiki;
 use kuchiki::traits::TendrilSink;
@@ -102,6 +102,7 @@ pub fn html_form(html: &str) -> Result<HashMap<String, String>> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use maplit::hashmap;
 
     #[test]
     fn test_html_select() {

sn0int-std/src/lib.rs

@@ -0,0 +1,26 @@
+#[macro_use] extern crate serde_derive;
+use hlua_badtouch as hlua;
+
+pub mod blobs;
+pub mod crt;
+pub mod crypto;
+mod errors;
+pub mod engine;
+pub mod geo;
+pub mod geoip;
+pub mod gfx;
+pub mod html;
+pub mod json;
+pub mod lazy;
+pub mod mqtt;
+pub mod psl;
+pub mod ratelimits;
+pub mod sockets;
+pub mod web;
+pub mod websockets;
+pub mod xml;
+
+#[cfg(test)]
+fn test_init() {
+    let _ = env_logger::builder().is_test(true).try_init();
+}

sn0int-std/src/mqtt.rs

@@ -0,0 +1,228 @@
+use chrootable_https::DnsResolver;
+use crate::errors::*;
+use crate::hlua::AnyLuaValue;
+use mqtt::packet::VariablePacketError;
+use crate::json::LuaJsonValue;
+use crate::sockets::{Stream, SocketOptions};
+use mqtt::{TopicFilter, QualityOfService};
+use mqtt::control::ConnectReturnCode;
+use mqtt::control::fixed_header::FixedHeaderError;
+use mqtt::encodable::{Encodable, Decodable};
+use mqtt::packet::{Packet, VariablePacket, ConnectPacket, SubscribePacket, PingreqPacket};
+use std::convert::TryFrom;
+use std::io;
+use std::net::SocketAddr;
+use url::Url;
+
+#[derive(Debug, Default, Deserialize)]
+pub struct MqttOptions {
+    pub username: Option<String>,
+    pub password: Option<String>,
+
+    pub proxy: Option<SocketAddr>,
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
+}
+
+impl MqttOptions {
+    pub fn try_from(x: AnyLuaValue) -> Result<MqttOptions> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub struct MqttClient {
+    stream: Stream,
+}
+
+impl MqttClient {
+    pub fn negotiate(stream: Stream, options: &MqttOptions) -> Result<MqttClient> {
+        let mut client = MqttClient {
+            stream,
+        };
+
+        let mut pkt = ConnectPacket::new("MQTT", "sn0int");
+        pkt.set_user_name(options.username.clone());
+        pkt.set_password(options.password.clone());
+
+        /*
+        if let Some(keep_alive) = msg.keep_alive {
+            packet.set_keep_alive(keep_alive);
+        }
+        */
+
+        client.send(pkt.into())?;
+        let pkt = client.recv()?;
+
+        if let VariablePacket::ConnackPacket(pkt) = pkt {
+            let code = pkt.connect_return_code();
+            if code == ConnectReturnCode::ConnectionAccepted {
+                Ok(client)
+            } else {
+                bail!("MQTT negotiation failed: {:?}", code);
+            }
+        } else {
+            bail!("Expected ConnAck, received {:?}", pkt);
+        }
+    }
+
+    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &MqttOptions) -> Result<MqttClient> {
+        let tls = match url.scheme() {
+            "mqtt" => false,
+            "mqtts" => true,
+            _ => bail!("Invalid mqtt protocol"),
+        };
+
+        let host = url.host_str()
+            .ok_or_else(|| format_err!("Missing host in url"))?;
+
+        let port = match (url.port(), tls) {
+            (Some(port), _) => port,
+            (None, true) => 8883,
+            (None, false) => 1883,
+        };
+
+
+        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
+            tls,
+            sni_value: None,
+            disable_tls_verify: false,
+            proxy: options.proxy,
+
+            connect_timeout: options.connect_timeout,
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+        })?;
+
+        Self::negotiate(stream, options)
+    }
+
+    fn send(&mut self, pkt: VariablePacket) -> Result<()> {
+        debug!("Sending mqtt packet: {:?}", pkt);
+        pkt.encode(&mut self.stream)?;
+        Ok(())
+    }
+
+    fn recv(&mut self) -> std::result::Result<VariablePacket, VariablePacketError> {
+        let pkt = VariablePacket::decode(&mut self.stream)?;
+        debug!("Received mqtt packet: {:?}", pkt);
+        Ok(pkt)
+    }
+
+    pub fn subscribe(&mut self, topic: &str, qos: u8) -> Result<()> {
+        let filter = TopicFilter::new(topic)?;
+
+        let qos = match qos {
+            0 => QualityOfService::Level0,
+            1 => QualityOfService::Level1,
+            2 => QualityOfService::Level2,
+            _ => bail!("Invalid QoS level: {}", qos),
+        };
+
+        let pkt = SubscribePacket::new(1, vec![(filter, qos)]);
+        self.send(pkt.into())?;
+
+        let pkt = self.recv()?;
+        if let VariablePacket::SubackPacket(_pkt) = pkt {
+            Ok(())
+        } else {
+            bail!("Expected SubAck, received {:?}", pkt);
+        }
+    }
+
+    pub fn recv_pkt(&mut self) -> Result<Option<Pkt>> {
+        match self.recv() {
+            Ok(pkt) => Ok(Some(Pkt::try_from(pkt)?)),
+            Err(VariablePacketError::IoError(err)) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(err) => Err(Error::from(err))
+        }
+    }
+
+    pub fn ping(&mut self) -> Result<()> {
+        let pkt = PingreqPacket::new();
+        self.send(pkt.into())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum Pkt {
+    #[serde(rename="publish")]
+    Publish(Publish),
+    #[serde(rename="pong")]
+    Pong,
+}
+
+impl Pkt {
+    pub fn to_lua(&self) -> Result<AnyLuaValue> {
+        let v = serde_json::to_value(&self)?;
+        let v = LuaJsonValue::from(v).into();
+        Ok(v)
+    }
+}
+
+impl TryFrom<VariablePacket> for Pkt {
+    type Error = Error;
+
+    fn try_from(pkt: VariablePacket) -> Result<Pkt> {
+        match pkt {
+            VariablePacket::ConnectPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::ConnackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PublishPacket(pkt) => Ok(Pkt::Publish(Publish {
+                topic: pkt.topic_name().to_string(),
+                body: pkt.payload(),
+            })),
+            VariablePacket::PubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubrecPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubrelPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubcompPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PingreqPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PingrespPacket(_) => Ok(Pkt::Pong),
+            VariablePacket::SubscribePacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::SubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::UnsubscribePacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::UnsubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::DisconnectPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Publish {
+    pub topic: String,
+    pub body: Vec<u8>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrootable_https::dns::Resolver;
+
+    fn connect() -> Result<MqttClient> {
+        let resolver = Resolver::from_system_v4().unwrap();
+        let url = "mqtt://mqtt.winkekatze24.de".parse()?;
+        MqttClient::connect(&resolver, url, &MqttOptions::default())
+    }
+
+    #[test]
+    #[ignore]
+    fn test_connect() {
+        connect().expect("Failed to setup connection");
+    }
+
+    // this test is too flaky
+    /*
+    #[test]
+    #[ignore]
+    fn test_subscribe() {
+        let mut c = connect().unwrap();
+        c.subscribe("#", 0).unwrap();
+    }
+    */
+}

sn0int-std/src/psl.rs

@@ -4,9 +4,7 @@ use chrootable_https::Client;
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use crate::paths;
 use crate::lazy::LazyInit;
-use crate::worker;
 
 
 #[derive(Debug, PartialEq)]
@@ -23,22 +21,24 @@ pub enum PslReader {
 }
 
 impl PslReader {
-    pub fn open_or_download() -> Result<PslReader> {
-        let path = Self::path()?;
+    pub fn open_or_download<F>(cache_dir: &Path, indicator: F) -> Result<PslReader>
+        where
+            F: Fn(Box<dyn Fn() -> Result<PslReader>>) -> Result<PslReader>
+    {
+        let path = Self::path(cache_dir)?;
         let reader = match Self::open_from(&path) {
             Ok(r) => r,
-            Err(_) => worker::spawn_fn("Downloading public suffix list", || {
+            Err(_) => indicator(Box::new(move || {
                 PslReader::download(&path, publicsuffix::LIST_URL)?;
                 Self::open_from(&path)
-            }, false)?,
+            }))?,
         };
         Ok(reader)
     }
 
-    pub fn open() -> Result<PslReader> {
-        let path = Self::path()?;
+    pub fn open(cache_dir: &Path) -> Result<PslReader> {
+        let path = Self::path(cache_dir)?;
         Self::open_from(&path)
-
     }
 
     pub fn open_from(path: &Path) -> Result<PslReader> {
@@ -46,7 +46,7 @@ impl PslReader {
         Ok(PslReader::Reader(file))
     }
 
-    pub fn path() -> Result<PathBuf> {
+    pub fn path(cache_dir: &Path) -> Result<PathBuf> {
         // use system path if exists
         let path = Path::new("/usr/share/publicsuffix/public_suffix_list.dat");
         if path.exists() {
@@ -54,13 +54,13 @@ impl PslReader {
         }
 
         // else, use local cache
-        let path = paths::cache_dir()?
+        let path = cache_dir
             .join("public_suffix_list.dat");
         Ok(path)
     }
 
     pub fn download(path: &Path, url: &str) -> Result<()> {
-        let client = Client::with_system_resolver()?;
+        let client = Client::with_system_resolver_v4()?;
         let resp = client.get(url)
             .wait_for_response()
             .context("http request failed")?;

sn0int-std/src/ratelimits.rs

@@ -0,0 +1,70 @@
+use chrono::prelude::*;
+use std::collections::HashMap;
+use std::result;
+use std::sync::mpsc;
+use std::time::Duration;
+
+pub type RatelimitSender = mpsc::Sender<result::Result<RatelimitResponse, String>>;
+
+pub struct Ratelimiter {
+    buckets: HashMap<String, Bucket>,
+}
+
+impl Ratelimiter {
+    pub fn new() -> Ratelimiter {
+        Ratelimiter {
+            buckets: HashMap::new(),
+        }
+    }
+
+    pub fn pass(&mut self, tx: RatelimitSender, key: &str, passes: u32, time: u32) {
+        let bucket = if let Some(bucket) = self.buckets.get_mut(key) {
+            bucket
+        } else {
+            let bucket = Bucket::new();
+            self.buckets.insert(key.to_string(), bucket);
+            self.buckets.get_mut(key).unwrap()
+        };
+        let reply = bucket.pass(passes as usize, time);
+        tx.send(Ok(reply)).unwrap();
+    }
+}
+
+struct Bucket {
+    passes: Vec<DateTime<Utc>>,
+}
+
+impl Bucket {
+    pub fn new() -> Bucket {
+        Bucket {
+            passes: Vec::new(),
+        }
+    }
+
+    pub fn pass(&mut self, passes: usize, time: u32) -> RatelimitResponse {
+        let now = Utc::now();
+        let time = chrono::Duration::milliseconds(time as i64);
+        let retain = now - time;
+        self.passes.retain(|x| *x >= retain);
+
+        if self.passes.len() >= passes {
+            if let Some(min) = self.passes.iter().min() {
+                let delay = time - (now - *min);
+                RatelimitResponse::Retry(delay.to_std().unwrap())
+            } else {
+                // This should never happen unless passes is zero
+                RatelimitResponse::Retry(Duration::from_millis(100))
+            }
+        } else {
+            let now = Utc::now();
+            self.passes.push(now);
+            RatelimitResponse::Pass
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum RatelimitResponse {
+    Retry(Duration),
+    Pass,
+}

sn0int-std/src/sockets/mod.rs

@@ -16,6 +16,7 @@ use std::io::BufRead;
 use std::net::SocketAddr;
 use std::net::TcpStream;
 use std::net::{IpAddr, Ipv4Addr};
+use std::time::Duration;
 
 mod tls;
 pub use self::tls::TlsData;
@@ -39,14 +40,21 @@ fn unwrap_socket(socket: tokio::net::TcpStream) -> Result<TcpStream> {
 pub struct SocketOptions {
     #[serde(default)]
     pub tls: bool,
-    sni_value: Option<String>,
+    pub sni_value: Option<String>,
     #[serde(default)]
-    disable_tls_verify: bool,
+    pub disable_tls_verify: bool,
     pub proxy: Option<SocketAddr>,
 
     // TODO: enable_sni (default to true)
+    // TODO: sni_name
     // TODO: cacert
-    // TODO: timeout
+
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
 }
 
 impl SocketOptions {
@@ -57,17 +65,97 @@ impl SocketOptions {
     }
 }
 
-#[derive(Debug)]
-pub struct Socket {
-    stream: BufStream<Stream>,
-    newline: String,
+impl SocketOptions {
+    pub fn apply(&self, stream: &Stream) -> Result<()> {
+        let socket = match stream {
+            Stream::Tcp(s) => s,
+            Stream::Tls(s) => s.get_ref(),
+        };
+        self.apply_tcp(socket)
+    }
+
+    pub fn apply_tcp(&self, socket: &TcpStream) -> Result<()> {
+        let read_timeout = self.read_timeout;
+        if read_timeout > 0 {
+            socket.set_read_timeout(Some(Duration::from_secs(read_timeout)))?;
+        }
+
+        let write_timeout = self.write_timeout;
+        if write_timeout > 0 {
+            socket.set_write_timeout(Some(Duration::from_secs(write_timeout)))?;
+        }
+
+        Ok(())
+    }
 }
 
-enum Stream {
+pub enum Stream {
     Tcp(TcpStream),
     Tls(rustls::StreamOwned<rustls::ClientSession, TcpStream>),
 }
 
+impl Stream {
+    pub fn connect_stream<R: DnsResolver>(resolver: &R, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+        let addrs = match host.parse::<IpAddr>() {
+            Ok(addr) => vec![addr],
+            Err(_) => resolver.resolve(host, RecordType::A)
+                .wait_for_response()?
+                .success()?,
+        };
+
+        let mut errors = Vec::new();
+
+        for addr in addrs {
+            match Stream::connect_addr(host, (addr, port).into(), &options) {
+                Ok(socket) => {
+                    return Ok(socket);
+                },
+                Err(err) => errors.push((addr, err)),
+            }
+        }
+
+        if errors.is_empty() {
+            bail!("no dns records found");
+        } else {
+            bail!("couldn't connect: {:?}", errors);
+        }
+    }
+
+    fn connect_addr(host: &str, addr: SocketAddr, options: &SocketOptions) -> Result<Stream> {
+        debug!("connecting to {}", addr);
+
+        let connect_timeout = options.connect_timeout;
+        let socket = if connect_timeout > 0 {
+            TcpStream::connect_timeout(&addr, Duration::from_secs(connect_timeout))?
+        } else {
+            TcpStream::connect(&addr)?
+        };
+        debug!("successfully connected to {:?}", addr);
+
+        options.apply_tcp(&socket)?;
+
+        tls::wrap_if_enabled(socket, host, options)
+    }
+
+    pub fn connect_socks5_stream(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+        debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
+
+        let addr = match host.parse::<Ipv4Addr>() {
+            Ok(ipaddr) => ProxyDest::Ipv4Addr(ipaddr),
+            _ => ProxyDest::Domain(host.to_string()),
+        };
+
+        let fut = socks5::connect(proxy, addr, port);
+
+        let mut rt = Runtime::new()?;
+        let socket = rt.block_on(fut)?;
+
+        let socket = unwrap_socket(socket)?;
+
+        tls::wrap_if_enabled(socket, host, options)
+    }
+}
+
 impl fmt::Debug for Stream {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
         match self {
@@ -102,6 +190,12 @@ impl Write for Stream {
     }
 }
 
+#[derive(Debug)]
+pub struct Socket {
+    stream: BufStream<Stream>,
+    newline: String,
+}
+
 impl Socket {
     fn new(stream: Stream) -> Socket {
         let stream = BufStream::new(stream);
@@ -112,60 +206,31 @@ impl Socket {
     }
 
     pub fn connect<R: DnsResolver>(resolver: &R, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
-        let addrs = match host.parse::<IpAddr>() {
-            Ok(addr) => vec![addr],
-            Err(_) => resolver.resolve(host, RecordType::A)
-                .wait_for_response()?
-                .success()?,
-        };
-
-        let mut errors = Vec::new();
-
-        for addr in addrs {
-            debug!("connecting to {}:{}", addr, port);
-            match TcpStream::connect((addr, port)) {
-                Ok(socket) => {
-                    debug!("successfully connected to {:?}", addr);
-                    return tls::wrap_if_enabled(socket, host, options);
-                },
-                Err(err) => errors.push((addr, err)),
-            }
-        }
-
-        if errors.is_empty() {
-            bail!("no dns records found");
-        } else {
-            bail!("couldn't connect: {:?}", errors);
-        }
+        let stream = Stream::connect_stream(resolver, host, port, options)?;
+        Ok(Socket::new(stream))
     }
 
     pub fn connect_socks5(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
-        debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
-
-        let addr = match host.parse::<Ipv4Addr>() {
-            Ok(ipaddr) => ProxyDest::Ipv4Addr(ipaddr),
-            _ => ProxyDest::Domain(host.to_string()),
-        };
-
-        let fut = socks5::connect(proxy, addr, port);
-
-        let mut rt = Runtime::new()?;
-        let socket = rt.block_on(fut)?;
-
-        let socket = unwrap_socket(socket)?;
-
-        tls::wrap_if_enabled(socket, host, options)
+        let stream = Stream::connect_socks5_stream(proxy, host, port, options)?;
+        Ok(Socket::new(stream))
     }
 
     pub fn upgrade_to_tls(self, options: &SocketOptions) -> Result<(Socket, TlsData)> {
         let stream = self.stream.into_inner()?;
 
-        match stream {
-            Stream::Tcp(stream) => tls::wrap(stream, "", options),
-            _ => bail!("Only tcp streams can be upgraded"),
+        if let Stream::Tcp(stream) = stream {
+            let (stream, tls) = tls::wrap(stream, "", options)?;
+            let socket = Socket::new(stream);
+            Ok((socket, tls))
+        } else {
+            bail!("Only tcp streams can be upgraded")
         }
     }
 
+    pub fn options(&self, options: &SocketOptions) -> Result<()> {
+        options.apply(self.stream.get_ref())
+    }
+
     pub fn send(&mut self, data: &[u8]) -> Result<()> {
         match str::from_utf8(&data) {
             Ok(data) => debug!("send: {:?}", data),
@@ -178,7 +243,12 @@ impl Socket {
 
     pub fn recv(&mut self) -> Result<Vec<u8>> {
         let mut buf = [0; 4096];
-        let n = self.stream.read(&mut buf)?;
+        let n = match self.stream.read(&mut buf) {
+            Ok(n) if n == 0 => bail!("Connection closed"),
+            Ok(n) => n,
+            Err(err) if err.kind() == io::ErrorKind::WouldBlock => 0,
+            Err(err) => return Err(err.into()),
+        };
         let data = buf[..n].to_vec();
         match str::from_utf8(&data) {
             Ok(data) => debug!("recv: {:?}", data),
@@ -249,6 +319,7 @@ impl Socket {
                 let available = match self.stream.fill_buf() {
                     Ok(n) => n,
                     Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
+                    Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(Vec::new()),
                     Err(e) => return Err(e.into())
                 };
 
@@ -293,7 +364,7 @@ mod tests {
     #[ignore]
     #[test]
     fn verify_tls_good() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let _sock = Socket::connect(&resolver, "badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -303,7 +374,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_good_request() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let mut sock = Socket::connect(&resolver, "badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -316,7 +387,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_expired() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let sock = Socket::connect(&resolver, "expired.badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -330,7 +401,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_1_1_1_1() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let _sock = Socket::connect(&resolver, "1.1.1.1", 443, &SocketOptions{
             tls: true,
             ..Default::default()

sn0int-std/src/sockets/tls.rs

@@ -2,14 +2,14 @@ use crate::errors::*;
 
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
-use rustls::{self, ClientConfig, Session, ClientSession, RootCertStore};
+use rustls::{self, ClientConfig, Session, ClientSession};
 
 use std::str;
 use std::result;
 use std::sync::Arc;
 use std::net::TcpStream;
 
-use super::{Socket, Stream, SocketOptions};
+use super::{Stream, SocketOptions};
 
 
 #[derive(Debug, Serialize)]
@@ -26,22 +26,22 @@ impl TlsData {
     }
 }
 
-pub fn wrap_if_enabled(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<Socket> {
+pub fn wrap_if_enabled(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<Stream> {
     if !options.tls {
         let stream = Stream::Tcp(stream);
-        return Ok(Socket::new(stream));
+        Ok(stream)
     } else {
         let (socket, _) = wrap(stream, host, options)?;
         Ok(socket)
     }
 }
 
-pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(Socket, TlsData)> {
-    let mut anchors = RootCertStore::empty();
-    anchors.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
-
+pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(Stream, TlsData)> {
     let mut config = ClientConfig::new();
-    config.root_store = anchors;
+    config
+        .root_store
+        .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
+    config.ct_logs = Some(&ct_logs::LOGS);
 
     if options.disable_tls_verify {
         info!("tls verification has been disabled");
@@ -73,14 +73,16 @@ fn get_dns_name(config: &mut ClientConfig, host: &str) -> webpki::DNSName {
     }
 }
 
-fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Socket, TlsData)> {
+fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Stream, TlsData)> {
     info!("starting tls handshake");
     if session.is_handshaking() {
-        session.complete_io(&mut stream)?;
+        session.complete_io(&mut stream)
+            .context("Failed to read reply to tls client hello")?;
     }
 
     if session.wants_write() {
-        session.complete_io(&mut stream)?;
+        session.complete_io(&mut stream)
+            .context("wants_write->complete_io failed")?;
     }
 
     let mut tls = TlsData {
@@ -106,7 +108,7 @@ fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Socket, T
     info!("successfully established tls connection");
     let stream = rustls::StreamOwned::new(session, stream);
     let stream = Stream::Tls(stream);
-    Ok((Socket::new(stream), tls))
+    Ok((stream, tls))
 }
 
 pub struct NoCertificateVerification {}

sn0int-std/src/web.rs

@@ -1,17 +1,17 @@
 pub use chrootable_https::{Client, HttpClient, Resolver, Response};
 
-use crate::blobs::Blob;
+use crate::blobs::{Blob, BlobState};
 use std::collections::{HashMap, HashSet};
 use std::time::Duration;
 use std::ops::Deref;
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
-use crate::engine::ctx::State;
 use serde_json;
 use rand::{Rng, thread_rng};
 use rand::distributions::Alphanumeric;
 use std::fmt;
 use std::net::SocketAddr;
+use std::sync::Arc;
 use serde::Serialize;
 use crate::engine::structs::LuaMap;
 use crate::json::LuaJsonValue;
@@ -37,6 +37,12 @@ pub fn url_set_qs<S: Serialize + fmt::Debug>(url: Uri, query: &S) -> Result<Uri>
         .map_err(Error::from)
 }
 
+pub trait WebState {
+    fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>>;
+
+    fn register_in_jar(&self, session: &str, key: String, value: String);
+}
+
 #[derive(Debug)]
 pub struct HttpSession {
     id: String,
@@ -66,6 +72,8 @@ pub struct RequestOptions {
     #[serde(default)]
     into_blob: bool,
     proxy: Option<SocketAddr>,
+    #[serde(default)]
+    binary: bool,
 }
 
 impl RequestOptions {
@@ -95,6 +103,7 @@ pub struct HttpRequest {
     timeout: Option<Duration>,
     into_blob: bool,
     proxy: Option<SocketAddr>,
+    binary: bool,
 }
 
 impl HttpRequest {
@@ -117,6 +126,7 @@ impl HttpRequest {
             timeout,
             into_blob: options.into_blob,
             proxy: options.proxy,
+            binary: options.binary,
         };
 
         if let Some(json) = options.json {
@@ -134,7 +144,7 @@ impl HttpRequest {
         request
     }
 
-    pub fn send(&self, state: &dyn State) -> Result<Response> {
+    pub fn send(&self, state: &dyn WebState) -> Result<Response> {
         let mut url = self.url.parse::<Uri>()?;
 
         // set query string
@@ -228,7 +238,9 @@ impl HttpRequest {
         Ok(res)
     }
 
-    pub fn response_to_lua(&self, state: &dyn State, res: Response) -> Result<LuaMap> {
+    pub fn response_to_lua<S>(&self, state: &S, res: Response) -> Result<LuaMap>
+        where S: WebState + BlobState
+    {
         // map result to LuaMap
         let mut resp = LuaMap::new();
         resp.insert_num("status", f64::from(res.status));
@@ -239,10 +251,16 @@ impl HttpRequest {
         }
         resp.insert("headers", headers);
 
+        if let Some(ipaddr) = res.ipaddr {
+            resp.insert_str("ipaddr", ipaddr.to_string());
+        }
+
         if self.into_blob {
             let blob = Blob::create(res.body);
             let id = state.register_blob(blob);
             resp.insert_str("blob", id);
+        } else if self.binary {
+            resp.insert_serde("binary", &res.body[..])?;
         } else {
             resp.insert_str("text", String::from_utf8_lossy(&res.body));
         }
@@ -250,7 +268,7 @@ impl HttpRequest {
         Ok(resp)
     }
 
-    fn register_cookies_on_state(session: &str, state: &dyn State, cookie: &str) {
+    fn register_cookies_on_state(session: &str, state: &dyn WebState, cookie: &str) {
         let mut key = String::new();
         let mut value = String::new();
         let mut in_key = true;

sn0int-std/src/websockets.rs

@@ -0,0 +1,150 @@
+use chrootable_https::DnsResolver;
+use crate::errors::*;
+use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
+use crate::sockets::{Stream, SocketOptions};
+use http::Request;
+use std::collections::HashMap;
+use std::net::SocketAddr;
+use std::io;
+use tungstenite::protocol::{self, Message};
+use url::Url;
+
+#[derive(Debug, Default, Deserialize)]
+pub struct WebSocketOptions {
+    pub headers: Option<HashMap<String, String>>,
+    pub proxy: Option<SocketAddr>,
+
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
+}
+
+impl WebSocketOptions {
+    pub fn try_from(x: AnyLuaValue) -> Result<WebSocketOptions> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub enum Event {
+    Text(String),
+    Binary(Vec<u8>),
+    Close,
+    Timeout,
+}
+
+pub struct WebSocket {
+    sock: protocol::WebSocket<Stream>,
+}
+
+impl WebSocket {
+    pub fn negotiate(stream: Stream, url: Url, headers: Option<&HashMap<String, String>>) -> Result<WebSocket> {
+        let mut req = Request::get(url.to_string()); // TODO: don't re-parse here
+
+        if let Some(headers) = headers {
+            for (k, v) in headers {
+                req = req.header(k, v);
+            }
+        }
+
+        let req = req.body(()).unwrap();
+
+        let (sock, _resp) = tungstenite::client::client(req, stream)?;
+        Ok(WebSocket {
+            sock,
+        })
+    }
+
+    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &WebSocketOptions) -> Result<WebSocket> {
+        let tls = match url.scheme() {
+            "ws" => false,
+            "wss" => true,
+            _ => bail!("Invalid websocket protocol"),
+        };
+
+        let host = url.host_str()
+            .ok_or_else(|| format_err!("Missing host in url"))?;
+
+        let port = match (url.port(), tls) {
+            (Some(port), _) => port,
+            (None, true) => 443,
+            (None, false) => 80,
+        };
+
+        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
+            tls,
+            sni_value: None,
+            disable_tls_verify: false,
+            proxy: options.proxy,
+
+            connect_timeout: options.connect_timeout,
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+        })?;
+        Self::negotiate(stream, url, options.headers.as_ref())
+    }
+
+    pub fn options(&self, options: &WebSocketOptions) -> Result<()> {
+        let mut o = SocketOptions::default();
+        o.read_timeout = options.read_timeout;
+        o.write_timeout = options.write_timeout;
+        o.apply(self.sock.get_ref())
+    }
+
+    fn read_msg(&mut self) -> Result<Event> {
+        loop {
+            let msg = match self.sock.read_message() {
+                Ok(Message::Text(body)) => Event::Text(body),
+                Ok(Message::Binary(body)) => Event::Binary(body),
+                Ok(Message::Ping(ping)) => {
+                    self.sock.write_message(Message::Pong(ping))?;
+                    continue;
+                },
+                Ok(Message::Pong(_)) => continue, // this should never happen
+                Ok(Message::Close(_)) => Event::Close,
+                Err(tungstenite::Error::ConnectionClosed) => Event::Close,
+                Err(tungstenite::Error::AlreadyClosed) => Event::Close,
+                Err(tungstenite::Error::Io(err)) if err.kind() == io::ErrorKind::WouldBlock => Event::Timeout,
+                Err(err) => return Err(err.into()),
+            };
+            return Ok(msg);
+        }
+    }
+
+    pub fn read_text(&mut self) -> Result<Option<String>> {
+        match self.read_msg()? {
+            Event::Text(text) => Ok(Some(text)),
+            Event::Binary(_) => bail!("Unexpected message type: binary"),
+            Event::Close => bail!("Connection closed"),
+            Event::Timeout => Ok(None),
+        }
+    }
+
+    pub fn read_binary(&mut self) -> Result<Option<Vec<u8>>> {
+        match self.read_msg()? {
+            Event::Text(_) => bail!("Unexpected message type: text"),
+            Event::Binary(body) => Ok(Some(body)),
+            Event::Close => bail!("Connection closed"),
+            Event::Timeout => Ok(None),
+        }
+    }
+
+    fn write_msg(&mut self, msg: Message) -> Result<()> {
+        self.sock.write_message(msg)?;
+        self.sock.write_pending()?;
+        Ok(())
+    }
+
+    pub fn write_text(&mut self, text: String) -> Result<()> {
+        self.write_msg(Message::Text(text))
+    }
+
+    pub fn write_binary(&mut self, binary: Vec<u8>) -> Result<()> {
+        self.write_msg(Message::Binary(binary))
+    }
+}

sn0int-std/src/xml.rs

@@ -124,6 +124,7 @@ fn decode_raw(x: &str) -> Result<XmlDocument> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use maplit::hashmap;
 
     #[test]
     fn verify_xml_decode_empty() {

src/api.rs

@@ -18,18 +18,27 @@ pub struct Client {
     server: String,
     client: chrootable_https::Client<chrootable_https::Resolver>,
     session: Option<String>,
+    os_version: String,
 }
 
 impl Client {
     pub fn new(config: &Config) -> Result<Client> {
-        let client = match config.network.proxy {
-            Some(proxy) => chrootable_https::Client::with_socks5(proxy),
-            _ => chrootable_https::Client::with_system_resolver()?,
+        let client = if let Some(proxy) = config.network.proxy {
+            chrootable_https::Client::with_socks5(proxy)
+        } else {
+            chrootable_https::Client::with_system_resolver_v4()?
         };
+
+        let os_version = match os_version::detect() {
+            Ok(v) => v.to_string(),
+            Err(_) => String::from("unknown"),
+        };
+
         Ok(Client {
             server: config.core.registry.clone(),
             client,
             session: None,
+            os_version,
         })
     }
 
@@ -41,12 +50,21 @@ impl Client {
         thread_rng().sample_iter(&Alphanumeric).take(32).collect()
     }
 
+    #[inline]
+    fn user_agent(&self) -> String {
+        format!("{}, {}, {}",
+            web::default_user_agent(),
+            embedded_triple::get(),
+            self.os_version,
+        )
+    }
+
     pub fn request<T: DeserializeOwned + fmt::Debug>(&self, mut request: RequestBuilder, body: Body) -> Result<T> {
         if let Some(session) = &self.session {
             info!("Adding session token to request");
             request.header("Auth", session.as_str());
         }
-        request.header("User-Agent", web::default_user_agent());
+        request.header("User-Agent", self.user_agent());
 
         let request = request.body(body)?;
 

src/archive.rs

@@ -1,40 +0,0 @@
-use crate::errors::*;
-use tar;
-use libflate::gzip;
-use std::io;
-use std::fs::File;
-use std::path::Path;
-
-
-pub fn extract<R: io::Read, P: AsRef<Path>>(read: &mut R, filter: &str, target: P) -> Result<()> {
-    let file = gzip::Decoder::new(read)?;
-    let mut ar = tar::Archive::new(file);
-
-    for entry in ar.entries()? {
-        let mut entry = entry?;
-        let file_name = {
-            let path = entry.path()?;
-            path.file_name()
-                .ok_or_else(|| format_err!("Invalid path in archive"))?
-                .to_str()
-                .ok_or_else(|| format_err!("Filename is invalid utf8"))?
-                .to_owned()
-        };
-
-        debug!("Found in archive: {:?}", file_name);
-
-        if filter == file_name {
-            debug!("Extracting to {:?}", target.as_ref());
-
-            let mut target = File::create(target)?;
-            let n = io::copy(&mut entry, &mut target)?;
-            debug!("Wrote {:?} bytes", n);
-
-            return Ok(());
-        }
-
-        debug!("Skipping file");
-    }
-
-    bail!("Nothing in archive matched filter")
-}

src/args.rs

@@ -48,6 +48,9 @@ pub enum SubCommand {
     /// Search in the registry
     #[structopt(name="search")]
     Search(Search),
+    /// The sn0int package manager
+    #[structopt(name="pkg")]
+    Pkg(cmd::pkg_cmd::Args),
     /// Insert into the database
     #[structopt(name="add")]
     Add(cmd::add_cmd::Args),
@@ -57,6 +60,9 @@ pub enum SubCommand {
     /// Delete from the database
     #[structopt(name="delete")]
     Delete(cmd::delete_cmd::Args),
+    /// Query logged activity
+    #[structopt(name="activity")]
+    Activity(cmd::activity_cmd::Args),
     /// Include entities in the scope
     #[structopt(name="scope")]
     Scope(cmd::scope_cmd::Args),
@@ -66,6 +72,12 @@ pub enum SubCommand {
     /// Manage workspaces
     #[structopt(name="workspace")]
     Workspace(cmd::workspace_cmd::Args),
+    /// Calendar
+    #[structopt(name="cal")]
+    Cal(cmd::cal_cmd::Args),
+    /// Notify
+    #[structopt(name="notify")]
+    Notify(cmd::notify_cmd::Args),
     /// Verify blob storage for corrupt and dangling blobs
     #[structopt(name="fsck")]
     Fsck(cmd::fsck_cmd::Args),
@@ -75,6 +87,9 @@ pub enum SubCommand {
     /// Run a lua repl
     #[structopt(name="repl")]
     Repl,
+    /// Show paths of various file system locations
+    #[structopt(name="paths")]
+    Paths,
     /// Generate shell completions
     #[structopt(name="completions")]
     Completions(Completions),
@@ -115,6 +130,9 @@ pub struct Run {
     /// Narrow down targeted entities
     #[structopt(short="t", long="target")]
     pub target: Option<String>,
+    /// Dump the sandbox init message to stdout instead of running a child process
+    #[structopt(long="dump-sandbox-init-msg")]
+    pub dump_sandbox_init_msg: bool,
 }
 
 #[derive(Debug, StructOpt)]

src/auth.rs

@@ -11,7 +11,7 @@ use crate::term;
 
 
 fn path() -> Result<PathBuf> {
-    let path = paths::data_dir()?;
+    let path = paths::sn0int_dir()?;
     Ok(path.join("auth"))
 }
 

src/blobs.rs

@@ -3,44 +3,13 @@ use crate::paths;
 use crate::worker::{EventWithCallback, Event2};
 use crate::workspaces::Workspace;
 
-use blake2::VarBlake2b;
-use digest::{Input, VariableOutput};
 use bytes::Bytes;
-use serde::ser::{Serialize, Serializer};
-use serde::de::{self, Deserialize, Deserializer};
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::mpsc;
 
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct Blob {
-    pub id: String,
-    pub bytes: Bytes,
-}
-
-impl Blob {
-    pub fn create(bytes: Bytes) -> Blob {
-        let id = Self::hash(&bytes);
-        Blob {
-            id,
-            bytes,
-        }
-    }
-
-    pub fn hash(bytes: &[u8]) -> String {
-        let mut h = VarBlake2b::new(32).unwrap();
-        h.input(bytes);
-        Self::encode_hash(&h.vec_result())
-    }
-
-    #[inline]
-    fn encode_hash(bytes: &[u8]) -> String {
-        let x = bs58::encode(bytes).into_string();
-        format!("{:0<44}", x)
-    }
-}
+pub use sn0int_std::blobs::Blob;
 
 impl EventWithCallback for Blob {
     type Payload = ();
@@ -51,30 +20,6 @@ impl EventWithCallback for Blob {
     }
 }
 
-impl Serialize for Blob {
-    #[inline]
-    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let s = base64::encode(&self.bytes);
-        serializer.serialize_str(&s)
-    }
-}
-
-impl<'de> Deserialize<'de> for Blob {
-    #[inline]
-    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let s = String::deserialize(deserializer)?;
-        let bytes = base64::decode(&s)
-            .map_err(de::Error::custom)?;
-        Ok(Blob::create(Bytes::from(bytes)))
-    }
-}
-
 pub struct BlobStorage {
     path: PathBuf,
 }
@@ -155,7 +100,6 @@ impl BlobStorage {
 mod tests {
     use super::*;
     use tempfile;
-    use serde_json;
 
     #[inline]
     fn blob() -> (Bytes, Blob) {
@@ -163,15 +107,6 @@ mod tests {
         (bytes.clone(), Blob::create(bytes))
     }
 
-    #[test]
-    fn verify_create_blob() {
-        let (bytes, blob) = blob();
-        assert_eq!(blob, Blob {
-            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
-            bytes,
-        });
-    }
-
     #[test]
     fn test_blobstorage_save() {
         let dir = tempfile::tempdir().expect("tempdir");
@@ -204,38 +139,4 @@ mod tests {
 
         assert!(result.is_err());
     }
-
-    #[test]
-    fn test_serialize() {
-        let (_, blob) = blob();
-        let json = serde_json::to_string(&blob).unwrap();
-        assert_eq!(&json, "\"YXNkZg==\"");
-    }
-
-    #[test]
-    fn test_deserialize() {
-        let (_, blob1) = blob();
-        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
-        assert_eq!(blob1, blob2);
-    }
-
-    #[test]
-    fn test_hash_encoding() {
-        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
-            .into_vec()
-            .unwrap();
-        let x = Blob::encode_hash(&x);
-        assert_eq!(x.len(), 44);
-        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
-    }
-
-    #[test]
-    fn test_hash_encoding_padding() {
-        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
-            .into_vec()
-            .unwrap();
-        let x = Blob::encode_hash(&x);
-        assert_eq!(x.len(), 44);
-        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
-    }
 }

src/cal/date.rs

@@ -0,0 +1,397 @@
+use crate::errors::*;
+
+use chrono::Utc;
+use chrono::prelude::*;
+use crate::cal::{ActivityGrade, DateArg};
+use crate::models::*;
+use std::collections::HashMap;
+use std::collections::VecDeque;
+
+const MONTH_LINES: i32 = 7;
+
+fn merge_months(ctx: &DateContext, months: &[DateSpec]) -> String {
+    let mut months = months.iter()
+        .map(|ds| {
+            let month = ds.to_term_string(ctx);
+            month.lines()
+                .map(String::from)
+                .collect::<VecDeque<_>>()
+        })
+        .collect::<Vec<_>>();
+
+    let mut out = String::new();
+    for i in 0..=MONTH_LINES {
+        let mut first = true;
+        for m in &mut months {
+            if !first {
+                out.push_str("   ");
+            }
+            if let Some(line) = m.pop_front() {
+                out.push_str(&line);
+            } else {
+                out.push_str(&" ".repeat(21));
+            }
+            first = false;
+        }
+        if i < MONTH_LINES {
+            out.push('\n');
+        }
+    }
+    out
+}
+
+fn chunk_months(ctx: &DateContext, months: &[DateSpec]) -> String {
+    months
+        .chunks(3)
+        .map(|m| merge_months(ctx, m))
+        .fold(String::new(), |a, b| {
+            if a.is_empty() {
+                a + &b
+            } else {
+                a + "\n" + &b
+            }
+        })
+}
+
+fn days_in_month(year: i32, month: u32) -> i64 {
+    let start = Utc.ymd(year, month, 1);
+    let end = if month == 12 {
+        Utc.ymd(year + 1, 1, 1)
+    } else {
+        Utc.ymd(year, month + 1, 1)
+    };
+    end.signed_duration_since(start).num_days()
+}
+
+fn setup_graph_map(events: &[Activity]) -> (HashMap<NaiveDate, u64>, u64) {
+    debug!("Found {} events in selected range", events.len());
+
+    let mut cur = None;
+    let mut ctr = 0;
+    let mut max = 0;
+
+    let mut map = HashMap::new();
+    for event in events {
+        let date = event.time.date();
+        if let Some(cur) = cur.as_mut() {
+            if date == *cur {
+                ctr += 1;
+            } else {
+                if ctr > max {
+                    max = ctr;
+                }
+                map.insert(cur.clone(), ctr);
+                *cur = date;
+                ctr = 1;
+            }
+        } else {
+            cur = Some(date);
+            ctr = 1;
+        }
+    }
+
+    if ctr > 0 {
+        if let Some(cur) = cur.take() {
+            if ctr > max {
+                max = ctr;
+            }
+            map.insert(cur, ctr);
+        }
+    }
+
+    debug!("Maximum events per day is {}", max);
+
+    (map, max)
+}
+
+pub struct DateContext {
+    events: HashMap<NaiveDate, u64>,
+    max: u64,
+    today: NaiveDate,
+}
+
+impl DateContext {
+    pub fn new(events: &[Activity], today: NaiveDate) -> DateContext {
+        let (events, max) = setup_graph_map(&events);
+        DateContext {
+            events,
+            max,
+            today,
+        }
+    }
+
+    #[inline]
+    fn is_today(&self, date: &NaiveDate) -> bool {
+        self.today == *date
+    }
+
+    #[inline]
+    fn is_future(&self, date: &NaiveDate) -> bool {
+        self.today < *date
+    }
+
+    fn activity_for_day(&self, date: &NaiveDate) -> ActivityGrade {
+        if let Some(events) = self.events.get(date) {
+            ActivityGrade::from_ratio(*events, self.max)
+        } else {
+            ActivityGrade::None
+        }
+    }
+}
+
+pub enum DateSpec {
+    Year(i32),
+    YearMonth((i32, u32)),
+    YearMonthContext((i32, u32, u32)),
+}
+
+impl DateSpec {
+    pub fn from_args(args: &[DateArg], context: Option<u32>) -> Result<DateSpec> {
+        if args.len() > 2 {
+            bail!("Too many datespec args");
+        }
+
+        let today = Utc::today();
+        let ds = match (args.get(0), args.get(1), context) {
+            (None, _, None) => DateSpec::YearMonth((today.year(), today.month())),
+            (None, _, Some(context)) => DateSpec::YearMonthContext((today.year(), today.month(), context)),
+
+            (Some(DateArg::Month(month)), None, None) => DateSpec::YearMonth((today.year(), *month)),
+            (Some(DateArg::Num(year)), None, None) => DateSpec::Year(*year),
+            (Some(DateArg::Month(month)), Some(DateArg::Num(year)), None) => DateSpec::YearMonth((*year, *month)),
+            (Some(DateArg::Num(year)), Some(DateArg::Month(month)), None) => DateSpec::YearMonth((*year, *month)),
+
+            (Some(DateArg::Month(month)), None, Some(context)) => DateSpec::YearMonthContext((today.year(), *month, context)),
+            (Some(DateArg::Month(month)), Some(DateArg::Num(year)), Some(context)) => DateSpec::YearMonthContext((*year, *month, context)),
+            (Some(DateArg::Num(year)), Some(DateArg::Month(month)), Some(context)) => DateSpec::YearMonthContext((*year, *month, context)),
+            _ => bail!("Combination of datespec args is invalid"),
+        };
+        Ok(ds)
+    }
+
+    pub fn start(&self) -> NaiveDate {
+        match self {
+            DateSpec::Year(year) => NaiveDate::from_ymd(*year, 1, 1),
+            DateSpec::YearMonth((year, month)) => NaiveDate::from_ymd(*year, *month, 1),
+            DateSpec::YearMonthContext((year, month, context)) => {
+                let mut year = *year - (*context / 12) as i32;
+                let context = context % 12;
+                let month = if context >= *month {
+                    year -= 1;
+                    12 - context + month
+                } else {
+                    month - context
+                };
+                NaiveDate::from_ymd(year, month, 1)
+            },
+        }
+    }
+
+    pub fn end(&self) -> NaiveDate {
+        match self {
+            DateSpec::Year(year) => NaiveDate::from_ymd(year + 1, 1, 1),
+            DateSpec::YearMonth((year, month)) => {
+                let (year, month) = if *month == 12 {
+                    (*year + 1, 1)
+                } else {
+                    (*year, *month + 1)
+                };
+                NaiveDate::from_ymd(year, month, 1)
+            },
+            DateSpec::YearMonthContext((year, month, _context)) => {
+                let (year, month) = if *month == 12 {
+                    (*year + 1, 1)
+                } else {
+                    (*year, *month + 1)
+                };
+                NaiveDate::from_ymd(year, month, 1)
+            },
+        }
+    }
+
+    pub fn to_term_string(&self, ctx: &DateContext) -> String {
+        match self {
+            DateSpec::Year(year) => {
+                let months = (1..=12)
+                    .map(|month| DateSpec::YearMonth((*year, month)))
+                    .collect::<Vec<_>>();
+                chunk_months(ctx, &months)
+            },
+            DateSpec::YearMonth((year, month)) => {
+                let mut w = String::new();
+
+                let start = Utc.ymd(*year, *month, 1);
+                let days = days_in_month(*year, *month) as u32;
+
+                w.push_str(&format!("{:^21}\n", start.format("%B %Y")));
+                w.push_str(" Su Mo Tu We Th Fr Sa\n");
+
+                let mut cur_week_day = start.weekday();
+                let week_progress = cur_week_day.num_days_from_sunday() as usize;
+                w.push_str(&"   ".repeat(week_progress));
+
+                let mut week_written = week_progress * 3;
+                for cur_day in 1..=days {
+                    let date = NaiveDate::from_ymd(*year, *month, cur_day);
+
+                    if !ctx.is_future(&date) {
+                        let activity = ctx.activity_for_day(&date);
+                        w.push_str(activity.as_term_str());
+                    }
+
+                    if ctx.is_today(&date) {
+                        w.push_str("\x1b[1m#");
+                    } else {
+                        w.push(' ');
+                    }
+                    w.push_str(&format!("{:2}", cur_day));
+                    week_written += 3;
+                    w.push_str("\x1b[0m");
+
+                    // detect end of the week
+                    if cur_week_day == Weekday::Sat {
+                        if cur_day != days {
+                            w.push('\n');
+                        }
+                        week_written = 0;
+                    }
+
+                    cur_week_day = cur_week_day.succ();
+                }
+                if week_written != 0 {
+                    w.push_str(&" ".repeat(21 - week_written));
+                }
+
+                w
+            }
+            DateSpec::YearMonthContext((_year, _month, context)) => {
+                let start = self.start();
+                let mut year = start.year();
+                let mut month = start.month();
+
+                let mut months = Vec::new();
+
+                for _ in 0..=*context {
+                    months.push(DateSpec::YearMonth((year, month)));
+
+                    if month == 12 {
+                        year += 1;
+                        month = 1;
+                    } else {
+                        month += 1;
+                    }
+                }
+
+                chunk_months(ctx, &months)
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn context() -> DateContext {
+        DateContext {
+            events: HashMap::new(),
+            max: 0,
+            today: NaiveDate::from_ymd(2020, 05, 30),
+        }
+    }
+
+    #[test]
+    fn test_days_in_month_2020_05() {
+        let days = days_in_month(2020, 05);
+        assert_eq!(days, 31);
+    }
+
+    #[test]
+    fn test_days_in_month_2020_04() {
+        let days = days_in_month(2020, 04);
+        assert_eq!(days, 30);
+    }
+
+    #[test]
+    fn test_days_in_month_2020_02() {
+        let days = days_in_month(2020, 02);
+        assert_eq!(days, 29);
+    }
+
+    #[test]
+    fn small_max_activity_0() {
+        let events = HashMap::new();
+        let ctx = DateContext {
+            events,
+            max: 0,
+            today: NaiveDate::from_ymd(2020, 06, 06),
+        };
+        let grade = ctx.activity_for_day(&NaiveDate::from_ymd(2020, 06, 06));
+        assert_eq!(grade, ActivityGrade::None);
+    }
+
+    #[test]
+    fn test_datespec_year_month() {
+        let ds = DateSpec::YearMonth((2020, 05));
+        let out = ds.to_term_string(&context());
+        assert_eq!(out, "      May 2020       
+ Su Mo Tu We Th Fr Sa
+               \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m\u{1b}[1m#30\u{1b}[0m
+ 31\u{1b}[0m                  ");
+    }
+
+    #[test]
+    fn test_datespec_year_month_ends_on_sat() {
+        let ds = DateSpec::YearMonth((2020, 10));
+        let out = ds.to_term_string(&context());
+        assert_eq!(out, "    October 2020     
+ Su Mo Tu We Th Fr Sa
+              1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m
+  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m
+ 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m
+ 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m
+ 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m");
+    }
+
+    #[test]
+    fn test_datespec_year() {
+        let ds = DateSpec::Year(2020);
+        let out = ds.to_term_string(&context());
+        assert_eq!(out, "    January 2020            February 2020            March 2020      
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+         \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m                     \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 30\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 31\u{1b}[0m      \u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 30\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 31\u{1b}[0m            
+                                                                     
+     April 2020               May 2020                June 2020      
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+         \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m                  \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m        1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m     7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m    14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m    21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 30\u{1b}[0m         \u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m\u{1b}[1m#30\u{1b}[0m    28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m            
+                         31\u{1b}[0m                                          
+      July 2020              August 2020           September 2020    
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+           1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m                       1\u{1b}[0m           1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m
+  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m     2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m     6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m
+ 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m     9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m    13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m
+ 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m    16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m    20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m
+ 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m       23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m    27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m         
+                         30\u{1b}[0m 31\u{1b}[0m                                       
+    October 2020            November 2020           December 2020    
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+              1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m     1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m           1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m
+  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m     8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m     6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m
+ 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m    15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m    13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m
+ 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m    22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m    20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m
+ 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m    29\u{1b}[0m 30\u{1b}[0m                   27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m      
+                                                                     ");
+    }
+}

src/cal/mod.rs

@@ -0,0 +1,166 @@
+use crate::errors::*;
+use std::str::FromStr;
+
+pub mod date;
+pub mod time;
+
+#[derive(Debug)]
+pub enum DateArg {
+    Month(u32),
+    Num(i32),
+}
+
+impl FromStr for DateArg {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<DateArg> {
+        let ds = match s.to_lowercase().as_str() {
+            "jan" | "january"   => DateArg::Month(1),
+            "feb" | "february"  => DateArg::Month(2),
+            "mar" | "march"     => DateArg::Month(3),
+            "apr" | "april"     => DateArg::Month(4),
+            "may"               => DateArg::Month(5),
+            "jun" | "june"      => DateArg::Month(6),
+            "jul" | "july"      => DateArg::Month(7),
+            "aug" | "august"    => DateArg::Month(8),
+            "sep" | "september" => DateArg::Month(9),
+            "oct" | "october"   => DateArg::Month(10),
+            "nov" | "november"  => DateArg::Month(11),
+            "dec" | "december"  => DateArg::Month(12),
+            _ => {
+                let num = s.parse::<i32>()
+                    .context("Input is not a month and not a number")?;
+                DateArg::Num(num)
+            },
+        };
+        Ok(ds)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum ActivityGrade {
+    None,
+    One,
+    Two,
+    Three,
+    Four,
+}
+
+impl ActivityGrade {
+    fn as_term_str(&self) -> &'static str {
+        match self {
+            ActivityGrade::None => "\x1b[97m\x1b[48;5;238m",
+            ActivityGrade::One => "\x1b[30m\x1b[48;5;148m",
+            ActivityGrade::Two => "\x1b[30m\x1b[48;5;71m",
+            ActivityGrade::Three => "\x1b[97m\x1b[48;5;34m",
+            ActivityGrade::Four => "\x1b[97m\x1b[48;5;22m",
+        }
+    }
+
+    pub fn from_ratio(num: u64, max: u64) -> ActivityGrade {
+        let max = max as f64;
+        let num = num as f64;
+        let step = max / 4.0;
+
+        let x = num / step;
+
+        if x <= 1.0 {
+            ActivityGrade::One
+        } else if x <= 2.0 {
+            ActivityGrade::Two
+        } else if x <= 3.0 {
+            ActivityGrade::Three
+        } else {
+            ActivityGrade::Four
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn small_max_activity_1() {
+        let grade = ActivityGrade::from_ratio(1, 1);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_2() {
+        let grade = ActivityGrade::from_ratio(2, 2);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_3() {
+        let grade = ActivityGrade::from_ratio(3, 3);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_4() {
+        let grade = ActivityGrade::from_ratio(4, 4);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_2_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 2);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_3_but_is_2() {
+        let grade = ActivityGrade::from_ratio(2, 3);
+        assert_eq!(grade, ActivityGrade::Three);
+    }
+
+    #[test]
+    fn small_max_activity_3_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 3);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_4_but_is_3() {
+        let grade = ActivityGrade::from_ratio(3, 4);
+        assert_eq!(grade, ActivityGrade::Three);
+    }
+
+    #[test]
+    fn small_max_activity_4_but_is_2() {
+        let grade = ActivityGrade::from_ratio(2, 4);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_4_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 4);
+        assert_eq!(grade, ActivityGrade::One);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_4() {
+        let grade = ActivityGrade::from_ratio(4, 5);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_3() {
+        let grade = ActivityGrade::from_ratio(3, 5);
+        assert_eq!(grade, ActivityGrade::Three);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_2() {
+        let grade = ActivityGrade::from_ratio(2, 5);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 5);
+        assert_eq!(grade, ActivityGrade::One);
+    }
+}

src/cal/time.rs

@@ -0,0 +1,194 @@
+use crate::errors::*;
+
+use chrono::Duration;
+use chrono::prelude::*;
+use crate::cal::{ActivityGrade, DateArg};
+use crate::models::*;
+use std::collections::HashMap;
+
+const MIN_PER_DAY: u32 = 1440;
+
+fn round_to_slice(time: &NaiveDateTime, slice_duration: u32) -> NaiveDateTime {
+    let date = time.date();
+    let hour = time.hour();
+    let mins = time.minute();
+    let slice = mins - (mins % slice_duration);
+    date.and_hms(hour, slice, 0)
+}
+
+fn setup_graph_map(events: &[Activity], slice_duration: u32) -> (HashMap<NaiveDateTime, u64>, u64) {
+    debug!("Found {} events in selected range", events.len());
+
+    let mut cur = None;
+    let mut ctr = 0;
+    let mut max = 0;
+
+    let mut map = HashMap::new();
+    for event in events {
+        let time = round_to_slice(&event.time, slice_duration);
+
+        if let Some(cur) = cur.as_mut() {
+            if time == *cur {
+                ctr += 1;
+            } else {
+                if ctr > max {
+                    max = ctr;
+                }
+                map.insert(cur.clone(), ctr);
+                *cur = time;
+                ctr = 1;
+            }
+        } else {
+            cur = Some(time);
+            ctr = 1;
+        }
+    }
+
+    if ctr > 0 {
+        if let Some(cur) = cur.take() {
+            if ctr > max {
+                max = ctr;
+            }
+            map.insert(cur, ctr);
+        }
+    }
+
+    debug!("Maximum events per slice is {}", max);
+
+    (map, max)
+}
+
+pub struct DateTimeContext {
+    events: HashMap<NaiveDateTime, u64>,
+    max: u64,
+    now: NaiveDateTime,
+    pub slice_width: u32,
+    pub slice_duration: u32,
+}
+
+impl DateTimeContext {
+    pub fn new(events: &[Activity], now: NaiveDateTime, slice_width: u32, slice_duration: u32) -> DateTimeContext {
+        let (events, max) = setup_graph_map(&events, slice_duration);
+        DateTimeContext {
+            events,
+            max,
+            now: round_to_slice(&now, slice_duration),
+            slice_width,
+            slice_duration,
+        }
+    }
+
+    #[inline]
+    fn is_future(&self, time: &NaiveDateTime) -> bool {
+        self.now < *time
+    }
+
+    #[inline]
+    fn slices_per_hour(&self) -> u32 {
+        60 / self.slice_duration
+    }
+
+    #[inline]
+    fn hour_width(&self) -> u32 {
+        self.slices_per_hour() * self.slice_width
+    }
+
+    fn activity_for_slice(&self, time: &NaiveDateTime) -> ActivityGrade {
+        if let Some(events) = self.events.get(time) {
+            ActivityGrade::from_ratio(*events, self.max)
+        } else {
+            ActivityGrade::None
+        }
+    }
+}
+
+pub struct DateTimeSpec {
+    start: NaiveDate,
+    end: NaiveDate,
+}
+
+impl DateTimeSpec {
+    pub fn from_args(args: &[DateArg], context: Option<u32>) -> Result<DateTimeSpec> {
+        let today = Utc::today().naive_utc();
+        if args.is_empty() {
+            let mut start = today;
+
+            if let Some(context) = context {
+                start = start.checked_sub_signed(Duration::days(context as i64))
+                    .ok_or_else(|| format_err!("Failed travel back in time"))?;
+            }
+
+            Ok(DateTimeSpec {
+                start,
+                end: today,
+            })
+        } else {
+            todo!()
+        }
+    }
+
+    pub fn start(&self) -> &NaiveDate {
+        &self.start
+    }
+
+    pub fn end(&self) -> &NaiveDate {
+        &self.end
+    }
+
+    pub fn to_term_string(&self, ctx: &DateTimeContext) -> String {
+        let mut w = String::new();
+
+        // add legend
+        w.push_str(&" ".repeat(11));
+        for x in 0..24 {
+            w.push_str(&format!("{:02}", x));
+
+            for i in 0..ctx.hour_width() {
+                if i >= 2 {
+                    w.push(' ');
+                }
+            }
+        }
+        w.push('\n');
+
+        // add days
+        let mut date = self.start;
+        while date <= self.end {
+            w.push_str(&date.format("%Y-%m-%d ").to_string());
+
+            let mut hours = 0;
+            let mut mins = 0;
+
+            for _ in 0..(MIN_PER_DAY / ctx.slice_duration) {
+                let time = date.and_hms(hours, mins, 0);
+
+                if !ctx.is_future(&time) {
+                    let activity = ctx.activity_for_slice(&time);
+                    w.push_str(activity.as_term_str());
+                } else {
+                    w.push_str("\x1b[0m");
+                }
+
+                for _ in 0..ctx.slice_width {
+                    w.push(' ');
+                }
+
+                mins += ctx.slice_duration;
+                if mins >= 60 {
+                    hours += 1;
+                    mins = 0;
+                }
+            }
+            w.push_str("\x1b[0m");
+
+            if date < self.end {
+                w.push('\n');
+            }
+
+            date = date.checked_add_signed(Duration::days(1))
+                .expect("Failed to get next day");
+        }
+
+        w
+    }
+}

src/cmd/activity_cmd.rs

@@ -0,0 +1,177 @@
+use crate::errors::*;
+
+use crate::cmd::Cmd;
+use crate::shell::Shell;
+use crate::models::*;
+use chrono::{Utc, NaiveDateTime, NaiveTime, Duration};
+use regex::Regex;
+use std::convert::TryFrom;
+use std::io;
+use std::str::FromStr;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+#[derive(Debug)]
+pub struct TimeSpec {
+    datetime: NaiveDateTime,
+}
+
+impl TimeSpec {
+    fn resolve(s: &str, now: NaiveDateTime) -> Result<Self> {
+        let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms(0, 0, 0));
+
+        let datetime = match s {
+            "today" => today,
+            "yesterday" => today - Duration::days(1),
+            s if s.ends_with(" ago") => {
+                let re = Regex::new(r"(\d+) ?(s|seconds?|m|min|minutes?|h|hours?|d|days?|w|weeks?|months?|y|years?) ago").unwrap();
+
+                let caps = re.captures(s)
+                    .ok_or_else(|| format_err!("Couldn't parse TimeSpec"))?;
+
+                let n = caps.get(1).unwrap().as_str()
+                    .parse::<i64>()
+                    .context("Failed to parse number in timespec")?;
+                let unit = caps.get(2).unwrap();
+
+                let duration = match unit.as_str() {
+                    "s" | "second" | "seconds" => Duration::seconds(n),
+                    "m" | "min" | "minute" | "minutes" => Duration::minutes(n),
+                    "h" | "hour" | "hours" => Duration::hours(n),
+                    "d" | "day" | "days" => Duration::days(n),
+                    "w" | "week" | "weeks" => Duration::days(n * 7),
+                    "month" | "months" => Duration::days(n * 31),
+                    "y" | "year" | "years" => Duration::days(n * 365),
+                    _ => unreachable!(),
+                };
+                now - duration
+            },
+            s => NaiveDateTime::from_str(s)?,
+        };
+
+        Ok(TimeSpec {
+            datetime,
+        })
+    }
+}
+
+impl FromStr for TimeSpec {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        let now = Utc::now().naive_utc();
+        Self::resolve(s, now)
+    }
+}
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Only query events for a given topic
+    #[structopt(short="t", long="topic")]
+    topic: Option<String>,
+    /// Only query events starting from that datetime
+    #[structopt(long="since")]
+    since: Option<TimeSpec>,
+    /// Only query events until this datetime
+    #[structopt(long="until")]
+    until: Option<TimeSpec>,
+    /// Try to select the previous event before --since as an initial state
+    #[structopt(short="i", long="initial")]
+    initial: bool,
+    /// Only query events that are tied to a location
+    #[structopt(short="l", long="location")]
+    location: bool,
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let filter = ActivityFilter {
+            topic: self.topic,
+            since: self.since.map(|t| t.datetime),
+            until: self.until.map(|t| t.datetime),
+            location: self.location,
+        };
+
+        let mut stdout = io::stdout();
+        let events = Activity::query(rl.db(), &filter)?;
+
+        if self.initial {
+            if let Some(first) = events.get(0) {
+                let previous = Activity::previous(rl.db(), first, &filter)?;
+                if let Some(previous) = previous {
+                    let mut previous = JsonActivity::try_from(previous)?;
+                    previous.initial = true;
+                    previous.write_to(&mut stdout)?;
+                }
+            }
+        }
+
+        for activity in events {
+            let activity = JsonActivity::try_from(activity)?;
+            activity.write_to(&mut stdout)?;
+        }
+
+        Ok(())
+    }
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn datetime() -> NaiveDateTime {
+        let date = chrono::NaiveDate::from_ymd(2020, 3, 14);
+        let time = chrono::NaiveTime::from_hms(16, 20, 23);
+        NaiveDateTime::new(date, time)
+    }
+
+    #[test]
+    fn test_today() {
+        let x = TimeSpec::resolve("today", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T00:00:00").unwrap());
+    }
+
+    #[test]
+    fn test_yesterday() {
+        let x = TimeSpec::resolve("yesterday", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-13T00:00:00").unwrap());
+    }
+
+    #[test]
+    fn test_20_min_ago() {
+        let x = TimeSpec::resolve("20min ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T16:00:23").unwrap());
+    }
+
+    #[test]
+    fn test_3_days_ago() {
+        let x = TimeSpec::resolve("3 days ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-11T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_1_week_ago() {
+        let x = TimeSpec::resolve("1w ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-07T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_3_months_ago() {
+        let x = TimeSpec::resolve("3 months ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2019-12-12T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_1_year_ago() {
+        let x = TimeSpec::resolve("1 year ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2019-03-15T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_exact_time() {
+        let x = TimeSpec::resolve("2020-03-14T16:20:23", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T16:20:23").unwrap());
+    }
+}

src/cmd/add_cmd.rs

@@ -11,6 +11,7 @@ use crate::utils;
 use crate::term;
 use std::fs;
 use std::net;
+use std::net::SocketAddr;
 use ipnetwork;
 use std::path::Path;
 use walkdir::WalkDir;
@@ -37,7 +38,7 @@ pub enum Target {
     #[structopt(name="ipaddr")]
     IpAddr(AddIpAddr),
     /// Insert url into the database
-    #[structopt(name="subdomain")]
+    #[structopt(name="url")]
     Url(AddUrl),
     /// Insert email into the database
     #[structopt(name="email")]
@@ -63,6 +64,9 @@ pub enum Target {
     /// Insert ip network into the database
     #[structopt(name="netblock")]
     Netblock(AddNetblock),
+    /// Insert port into the database
+    #[structopt(name="port")]
+    Port(AddPort),
     /// Insert a crypto currency address into the database
     #[structopt(name="cryptoaddr")]
     CryptoAddr(AddCryptoAddr),
@@ -83,6 +87,7 @@ impl Cmd for Args {
             Target::Breach(args) => args.insert(rl, self.dry_run),
             Target::Image(args) => args.insert(rl, self.dry_run),
             Target::Netblock(args) => args.insert(rl, self.dry_run),
+            Target::Port(args) => args.insert(rl, self.dry_run),
             Target::CryptoAddr(args) => args.insert(rl, self.dry_run),
         }
     }
@@ -555,6 +560,71 @@ impl IntoInsert for AddNetblock {
     }
 }
 
+#[derive(Debug, StructOpt)]
+pub struct AddPort {
+    protocol: Option<String>,
+    addr: Option<SocketAddr>,
+}
+
+impl IntoInsert for AddPort {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
+        let protocol = if let Some(protocol) = self.protocol {
+            protocol
+        } else {
+            utils::question("Protocol (tcp/udp)")?
+        };
+
+        let addr = if let Some(addr) = self.addr {
+            addr
+        } else {
+            let addr = utils::question("IP:Port")?;
+            addr.parse()?
+        };
+
+        let family = match addr.ip() {
+            net::IpAddr::V4(_) => "4",
+            net::IpAddr::V6(_) => "6",
+        };
+
+        let ip_addr_id = match rl.db().insert_struct(NewIpAddr {
+            family: family.to_string(),
+            value: addr.ip().to_string(),
+            continent: None,
+            continent_code: None,
+            country: None,
+            country_code: None,
+            city: None,
+            latitude: None,
+            longitude: None,
+            asn: None,
+            as_org: None,
+            description: None,
+            reverse_dns: None,
+            unscoped: false,
+        }, true)? {
+            Some((_, ip_addr_id)) => ip_addr_id,
+            _ => bail!("IpAddr is out out of scope"),
+        };
+
+        let value = format!("{}/{}", protocol, addr);
+
+        Ok(Insert::Port(NewPort {
+            ip_addr_id,
+            value,
+            ip_addr: addr.ip().to_string(),
+            port: addr.port() as i32,
+            protocol,
+            status: None,
+
+            banner: None,
+            service: None,
+            version: None,
+
+            unscoped: false,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddCryptoAddr {
     address: Option<String>,

src/cmd/cal_cmd.rs

@@ -0,0 +1,66 @@
+use crate::errors::*;
+
+use chrono::Utc;
+use crate::cal::DateArg;
+use crate::cal::date::{DateContext, DateSpec};
+use crate::cal::time::{DateTimeContext, DateTimeSpec};
+use crate::cmd::Cmd;
+use crate::models::*;
+use crate::shell::Shell;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Show additional months for context
+    #[structopt(short="C", long)]
+    context: Option<u32>,
+    /// Group events in 12 min slices
+    #[structopt(short="T", long, group = "view")]
+    time: bool,
+    /// Group events by hour
+    #[structopt(short="H", long, group = "view")]
+    hourly: bool,
+    args: Vec<DateArg>,
+}
+
+impl Cmd for Args {
+    #[inline]
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        if self.time || self.hourly {
+            let dts = DateTimeSpec::from_args(&self.args, self.context)
+                .context("Failed to parse date spec")?;
+            let filter = ActivityFilter {
+                topic: None,
+                since: Some(dts.start().and_hms(0, 0, 0)),
+                until: Some(dts.end().and_hms(23, 59, 59)),
+                location: false,
+            };
+            let events = Activity::query(rl.db(), &filter)?;
+
+            let (slice_width, slice_duration) = if self.hourly {
+                (3, 60)
+            } else {
+                (1, 12)
+            };
+
+            let ctx = DateTimeContext::new(&events, Utc::now().naive_utc(), slice_width, slice_duration);
+            println!("{}", dts.to_term_string(&ctx));
+        } else {
+            let ds = DateSpec::from_args(&self.args, self.context)
+                .context("Failed to parse date spec")?;
+            let filter = ActivityFilter {
+                topic: None,
+                since: Some(ds.start().and_hms(0, 0, 0)),
+                until: Some(ds.end().and_hms(23, 59, 59)),
+                location: false,
+            };
+            let events = Activity::query(rl.db(), &filter)?;
+            let ctx = DateContext::new(&events, Utc::today().naive_utc());
+            println!("{}", ds.to_term_string(&ctx));
+        }
+        Ok(())
+    }
+}

src/cmd/export_cmd.rs

@@ -3,6 +3,7 @@ use crate::errors::*;
 use crate::blobs::Blob;
 use crate::cmd::Cmd;
 use crate::db::ttl;
+use crate::models::*;
 use crate::shell::Shell;
 use serde_json;
 use serde::Serialize;
@@ -10,7 +11,6 @@ use std::io::{self, Write};
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use strum_macros::{EnumString, IntoStaticStr};
-use crate::models::*;
 
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
@@ -22,7 +22,7 @@ pub struct Args {
 
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
-        ttl::reap_expired(rl.db())?;
+        ttl::reap_expired(rl)?;
         match self.format {
             Format::Json => export::<JsonFormat>(rl),
             Format::JsonBlobs => export::<JsonBlobsFormat>(rl),

src/cmd/help_cmd.rs

@@ -16,9 +16,8 @@ pub fn run(_rl: &mut Shell, _args: &[String]) -> Result<()> {
     help("autoscope",   "Manage rules to automatically add entities to scope");
     help("delete",      "Delete entities from the database");
     help("keyring",     "Manage saved credentials");
-    help("mod",         "Manage installed modules");
+    help("pkg",         "Manage installed modules");
     help("noscope",     "Exclude entities from scope");
-    help("quickstart",  "Install all featured modules");
     help("run",         "Run the currently selected module");
     help("scope",       "Include entities in the scope again");
     help("select",      "Select entities from the database");

src/cmd/mod.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 use crate::shell::Shell;
+use crate::config::Config;
 
 pub trait Cmd: structopt::StructOpt + Sized {
     fn run(self, rl: &mut Shell) -> Result<()>;
@@ -11,9 +12,15 @@ pub trait Cmd: structopt::StructOpt + Sized {
     }
 }
 
+pub trait LiteCmd: structopt::StructOpt + Sized {
+    fn run(self, config: &Config) -> Result<()>;
+}
+
+pub mod activity_cmd;
 pub mod add_cmd;
 pub mod autonoscope_cmd;
 pub mod autoscope_cmd;
+pub mod cal_cmd;
 pub mod delete_cmd;
 pub mod export_cmd;
 pub mod fsck_cmd;
@@ -22,8 +29,9 @@ pub mod run_cmd;
 pub mod use_cmd;
 pub mod select_cmd;
 pub mod keyring_cmd;
-pub mod mod_cmd;
 pub mod noscope_cmd;
+pub mod notify_cmd;
+pub mod pkg_cmd;
 pub mod set_cmd;
 pub mod scope_cmd;
 pub mod target_cmd;

src/cmd/mod_cmd.rs

@@ -1,152 +0,0 @@
-use crate::errors::*;
-
-use crate::args;
-use crate::registry::{self, UpdateTask, Updater};
-use crate::shell::Shell;
-use crate::update::AutoUpdater;
-use crate::worker;
-use colored::Colorize;
-use sn0int_common::ModuleID;
-use std::fmt::Write;
-use std::sync::Arc;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
-
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
-pub struct Args {
-    #[structopt(subcommand)]
-    pub subcommand: SubCommand,
-}
-
-#[derive(Debug, StructOpt)]
-pub enum SubCommand {
-    /// List installed modules
-    #[structopt(name="list")]
-    List(List),
-    /// Install module from registry
-    #[structopt(name="install")]
-    Install(args::Install),
-    /// Search modules in registry
-    #[structopt(name="search")]
-    Search(args::Search),
-    /// Reload modules
-    #[structopt(name="reload")]
-    Reload(Reload),
-    /// Update modules
-    #[structopt(name="update")]
-    Update(Update),
-    /// Uninstall a module
-    #[structopt(name="uninstall")]
-    Uninstall(Uninstall),
-}
-
-#[derive(Debug, StructOpt)]
-pub struct List {
-    /// Only show modules with a specific input source
-    #[structopt(long="source")]
-    pub source: Option<String>,
-    /// List outdated modules
-    #[structopt(long="outdated")]
-    pub outdated: bool,
-}
-
-#[derive(Debug, StructOpt)]
-pub struct Reload {
-}
-
-#[derive(Debug, StructOpt)]
-pub struct Update {
-}
-
-#[derive(Debug, StructOpt)]
-pub struct Uninstall {
-    module: ModuleID,
-}
-
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    let config = rl.config().clone();
-
-    match args.subcommand {
-        SubCommand::List(list) => {
-            let autoupdate = AutoUpdater::load()?;
-
-            for module in rl.engine().list() {
-                if let Some(source) = &list.source {
-                    if !module.source_equals(&source) {
-                        continue;
-                    }
-                }
-
-                let canonical = module.canonical();
-
-                let mut out = String::new();
-                write!(&mut out, "{} ({})", canonical.green(),
-                                            module.version().yellow())?;
-                if autoupdate.is_outdated(&canonical) {
-                    write!(&mut out, " {}", "[outdated]".red())?;
-                } else if list.outdated {
-                    continue;
-                }
-                println!("{}", out);
-                println!("\t{}", module.description());
-            }
-        },
-        SubCommand::Install(install) => {
-            registry::run_install(install, &config)?;
-            // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
-        },
-        SubCommand::Search(search) => registry::run_search(rl.engine(), &search, &config)?,
-        SubCommand::Reload(_) => {
-            let current = rl.take_module()
-                            .map(|m| m.canonical());
-
-            rl.engine_mut().reload_modules()?;
-            rl.reload_module_cache();
-
-            if let Some(module) = current {
-                if let Ok(module) = rl.engine().get(&module).map(|x| x.to_owned()) {
-                    rl.set_module(module);
-                }
-            }
-        },
-        SubCommand::Update(_) => {
-            let mut autoupdate = AutoUpdater::load()?;
-            let updater = Arc::new(Updater::new(&config)?);
-
-            let modules = rl.engine().list()
-                .into_iter()
-                .filter_map(|module| {
-                    let canonical = module.canonical();
-
-                    if module.is_private() {
-                        debug!("{} is a private module, skipping", canonical);
-                        return None;
-                    }
-
-                    Some(UpdateTask::new(module.clone(), updater.clone()))
-                })
-                .collect::<Vec<_>>();
-
-            worker::spawn_multi(modules, |name| {
-                autoupdate.updated(&name);
-            }, 3)?;
-
-            autoupdate.save()?;
-
-            // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
-        },
-        SubCommand::Uninstall(uninstall) => {
-            let updater = Updater::new(&config)?;
-            updater.uninstall(&uninstall.module)?;
-            // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
-        },
-    }
-
-    Ok(())
-}

src/cmd/notify_cmd.rs

@@ -0,0 +1,99 @@
+use crate::errors::*;
+
+use crate::cmd::Cmd;
+use crate::engine::Module;
+// use crate::models::*;
+use crate::notify::{self, Notification};
+use crate::options::{self, Opt};
+use crate::shell::Shell;
+use crate::term;
+use sn0int_std::ratelimits::Ratelimiter;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Subcommand,
+}
+
+#[derive(Debug, StructOpt)]
+pub enum Subcommand {
+    /// Manually add a notification to the outbox
+    Send(SendArgs),
+    /// Show the current outbox
+    Outbox,
+    /// Execute a module directly instead of sending a message
+    Exec(ExecArgs),
+    /// Try to deliver all messages in our outbox
+    Deliver,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct SendArgs {
+    /// Evaluate the routing rules, but do not actually send a notification
+    #[structopt(short="n", long)]
+    pub dry_run: bool,
+    pub topic: String,
+    #[structopt(flatten)]
+    pub notification: Notification,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct ExecArgs {
+    pub module: String,
+    #[structopt(short="o", long="option")]
+    pub options: Vec<options::Opt>,
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
+    verbose: u64,
+    #[structopt(flatten)]
+    pub notification: Notification,
+}
+
+fn print_summary(module: &Module, sent: usize, errors: usize) {
+        let mut out = if sent == 1 {
+            String::from("Sent 1 notification")
+        } else {
+            format!("Sent {} notifications", sent)
+        };
+
+        out.push_str(&format!(" with {}", module.canonical()));
+
+        if errors > 0 {
+            out.push_str(&format!(" ({} errors)", errors));
+        }
+
+        term::info(&out);
+}
+
+fn send(args: SendArgs, rl: &mut Shell) -> Result<()> {
+    rl.signal_register().catch_ctrl();
+    notify::run_router(rl, &mut term::Term, &mut Ratelimiter::new(), args.dry_run, &args.topic, &args.notification)?;
+    rl.signal_register().reset_ctrlc();
+    Ok(())
+}
+
+fn exec(args: ExecArgs, rl: &mut Shell) -> Result<()> {
+    let module = rl.library().get(&args.module)?.clone();
+    let options = Opt::collect(&args.options);
+
+    rl.signal_register().catch_ctrl();
+    let errors = notify::exec(rl, &module, &mut Ratelimiter::new(), options, args.verbose, &args.notification)?;
+    rl.signal_register().reset_ctrlc();
+
+    print_summary(&module, 1, errors);
+    Ok(())
+}
+
+impl Cmd for Args {
+    #[inline]
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        match self.subcommand {
+            Subcommand::Send(args) => send(args, rl),
+            Subcommand::Outbox => todo!(),
+            Subcommand::Exec(args) => exec(args, rl),
+            Subcommand::Deliver => todo!(),
+        }
+    }
+}