Release v0.25.0

Update dependencies, fix #238

.dockerignore

@@ -1,6 +1,7 @@
 target
 Dockerfile
 .dockerignore
+docker-compose.yml
 docker
 docs
 ci

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: [kpcyrd]
+patreon: kpcyrd

.github/workflows/docker-release.yml

@@ -0,0 +1,58 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to GitHub Registry
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/kpcyrd/sn0int
+          tags: |
+            type=semver,pattern={{raw}}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: Login to Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push Docker images
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          file: Dockerfile
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+      -
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/docker.yml

@@ -0,0 +1,30 @@
+name: Docker
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+        - name: sn0int
+          file: Dockerfile
+        - name: registry
+          file: sn0int-registry/Dockerfile
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Build the Docker image
+      run: DOCKER_BUILDKIT=1 docker build -t ${{ matrix.build.name }} -f ${{ matrix.build.file }} .
+    - name: Test the Docker image
+      run: docker run --rm ${{ matrix.build.name }} --help
+
+    - name: Show Docker images
+      run: docker images

.github/workflows/rust.yml

@@ -0,0 +1,69 @@
+name: Rust
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 9 * * 1'
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install dependencies (apt)
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get install libsqlite3-dev libseccomp-dev libsodium-dev
+    - name: Install dependencies (brew)
+      if: matrix.os == 'macos-latest'
+      run: brew install pkg-config libsodium
+
+    - name: Build (sn0int)
+      run: cargo build --verbose
+    - name: Build (common)
+      run: cd sn0int-common && cargo build --verbose
+    - name: Build (std)
+      run: cd sn0int-std && cargo build --verbose
+    - name: Build (examples)
+      run: cargo build --verbose --examples
+
+    - name: Run tests (sn0int)
+      run: cargo test --verbose
+    - name: Run tests (sn0int, --ignored)
+      run: cargo test --verbose -- --ignored
+    - name: Run tests (common)
+      run: cd sn0int-common && cargo test --verbose
+    - name: Run tests (common, --ignored)
+      run: cd sn0int-common && cargo test --verbose -- --ignored
+    - name: Run tests (std)
+      run: cd sn0int-std && cargo test --verbose
+    - name: Run tests (std, --ignored)
+      run: cd sn0int-std && cargo test --verbose -- --ignored
+
+  notify:
+    # forks shouldn't notify
+    if: github.repository == 'kpcyrd/sn0int'
+    runs-on: ubuntu-latest
+    needs:
+    - build
+
+    steps:
+    - name: irc notify
+      uses: rectalogic/notify-irc@v1
+      with:
+        server: irc.hackint.org
+        channel: "#sn0int"
+        nickname: github-ci
+        message: '${{ github.repository }}#${{ github.run_id }}(${{ github.event_name }}): ${{ github.ref }}: tests completed: ${{ needs.build.result }} (https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})'

.travis.yml

@@ -1,58 +0,0 @@
-language: rust
-
-# upload takes too long on windows and gets killed due to inactivity
-#cache: cargo
-
-matrix:
-  include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker-registry
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  #- os: windows
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE="windows test"
-  - os: windows
-    rust: stable
-    env:
-    - BUILD_MODE="windows common"
-
-before_install:
-- ci/setup.sh "$TRAVIS_OS_NAME"
-script:
-- df -h
-- ci/run.sh $BUILD_MODE
-- df -h
-
-notifications:
-  irc:
-    channels:
-    - "ircs://irc.hackint.org:6697/#sn0int"
-    #on_success: change # default: always
-    #on_failure: always # default: always
-    use_notice: true

Cargo.toml

@@ -1,105 +1,104 @@
 [package]
 name = "sn0int"
-version = "0.11.0"
+version = "0.25.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
-edition = "2018"
-
-[profile.release]
-# skip lto to avoid compiler bug:
-# https://github.com/kpcyrd/sn0int/issues/77
-# https://github.com/rust-lang/rust/issues/58674
-opt-level = 1
-lto = false
-
-[badges]
-travis-ci = { repository = "kpcyrd/sn0int" }
+edition = "2021"
 
 [workspace]
-# rocket is broken and blocks the whole workspace, removing
-#members = ["sn0int-registry/sn0int-common",
-#           "sn0int-registry"]
-members = ["sn0int-registry/sn0int-common"]
+members = ["sn0int-common",
+           "sn0int-registry",
+           "sn0int-std"]
+
+[package.metadata.deb]
+extended-description = """\
+sn0int (pronounced /snoɪnt/) is a semi-automatic OSINT framework and package
+manager. It was built for IT security professionals and bug hunters to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations."""
+section = "utils"
+priority = "optional"
+depends = "$auto, publicsuffix"
+assets = [
+    ["target/release/sn0int", "usr/bin/", "755"],
+]
+
+[features]
+sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.6.0", path="sn0int-registry/sn0int-common" }
-rustyline = "3"
+sn0int-common = { version="0.13.0", path="sn0int-common" }
+sn0int-std = { version="=0.25.0", path="sn0int-std" }
+rustyline = "10.0"
 log = "0.4"
-env_logger = "0.6"
+env_logger = "0.9"
 hlua-badtouch = "0.4"
-structopt = "0.2"
+structopt = "0.3"
 failure = "0.1"
-rand = "0.6"
-colored = "1.6"
+rand = "0.8"
+colored = "2"
 lazy_static = "1.0"
 shellwords = "1.0"
-publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
+libsqlite3-sys = { version = "0.22.0", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
-dirs = "1.0"
-url = "1.7"
+dirs-next = "2.0"
+url = "2.0"
+percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.9"
-base64 = "0.10"
-kuchiki = "0.7.2"
-serde_urlencoded = "0.5"
-serde = "1.0"
-serde_derive = "1.0"
+chrootable-https = "0.16"
+data-encoding = "2.3.3"
+serde = { version = "1.0", features = ["derive"] }
+serde_urlencoded = "0.7"
 serde_json = "1.0"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.5"
 ctrlc = "3.1"
-opener = "0.3.0"
+opener = "0.5"
 separator = "0.4"
 maplit = "1.0.1"
-sloppy-rfc4880 = "0.1.2"
+sloppy-rfc4880 = "0.2"
 regex = "1.0"
 toml = "0.5"
-maxminddb = "0.13"
-tar = "0.4.17"
-libflate = "0.1.14"
 threadpool = "1.7"
-x509-parser = "0.4.0"
-der-parser = "1.1.0"
-nom = "4.1.1"
 atty = "0.2"
-bufstream = "0.1.4"
-tokio = "0.1.14"
-semver = "0.9"
+semver = "1"
 bytes = "0.4"
+bytesize = "1.0"
+ipnetwork = "0.18"
+strum = "0.24"
+strum_macros = "0.24"
+embedded-triple = "0.1.0"
+humansize = "1.1.0"
 
-digest = "0.8.0"
-hex = "0.3.1"
-bs58 = "0.2.2"
-blake2 = "0.8.0"
-md-5 = "0.8.0"
-sha-1 = "0.8.1"
-sha2 = "0.8.0"
+digest = "0.10"
+md-5 = "0.10"
+sha-1 = "0.10"
+sha2 = "0.10"
+sha3 = "0.10"
+hmac = "0.12"
 
-image = "0.21"
-kamadak-exif = "0.3.1"
 walkdir = "2.2"
-nude = "0.1.0"
+nude = "0.3"
+glob = "0.3.0"
+os-version = "0.2"
 
 [target.'cfg(target_os="linux")'.dependencies]
-caps = "0.3"
+caps = "0.5"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.11"
-nix = "0.13"
+syscallz = "0.16"
+nix = "0.24"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
-pledge = "0.3.1"
-unveil = "0.2.0"
+pledge = "0.4"
+unveil = "0.3"
 
 [dev-dependencies]
 #boxxy = { path = "../boxxy-rs" }
-boxxy = "0.9"
+boxxy = "0.13"
 tempfile = "3.0"
-
-#[patch.crates-io]
-#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }
-#rocket_contrib = { git = "https://github.com/SergioBenitez/Rocket.git", rev="c86f4312fb273c0380fb76a97bfb7fc227800542" }

Dockerfile

@@ -1,15 +1,18 @@
-FROM rust
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM rust:alpine3.15
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo build --release --verbose
-RUN strip target/release/sn0int
+RUN --mount=type=cache,target=/var/cache/buildkit \
+    CARGO_HOME=/var/cache/buildkit/cargo \
+    CARGO_TARGET_DIR=/var/cache/buildkit/target \
+    cargo build --release --locked --verbose && \
+    cp -v /var/cache/buildkit/target/release/sn0int /
+RUN strip /sn0int
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
-COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+FROM alpine:3.15
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
+COPY --from=0 /sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \
     XDG_CACHE_HOME=/cache

Makefile

@@ -1,19 +1,23 @@
 check:
-	(cd sn0int-registry/sn0int-common; cargo check)
+	(cd sn0int-common; cargo check)
 	(cd sn0int-registry; cargo check)
+	(cd sn0int-std; cargo check)
 	cargo check
 
 force-check:
-	(cd sn0int-registry/sn0int-common; touch src/lib.rs; cargo check)
+	(cd sn0int-common; touch src/lib.rs; cargo check)
 	(cd sn0int-registry; touch src/main.rs; cargo check)
+	(cd sn0int-std; touch src/lib.rs; cargo check)
 	touch src/lib.rs
 	cargo check
 
 test:
-	(cd sn0int-registry/sn0int-common; cargo test)
+	(cd sn0int-common; cargo test)
 	(cd sn0int-registry; cargo test)
-	cargo test
-	cargo test -- --ignored
+	(cd sn0int-std; cargo test)
+	(cd sn0int-std; cargo test -- --ignored)
+	cargo test --lib
+	cargo test --lib -- --ignored
 
 update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt

README.md

@@ -1,43 +1,47 @@
-# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
+# sn0int [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
-[travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
-[travis]:       https://travis-ci.org/kpcyrd/sn0int
 [crates-img]:   https://img.shields.io/crates/v/sn0int.svg
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
 [docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
 [irc-img]:      https://img.shields.io/badge/hackint-%23sn0int-blue.svg
 [irc]:          https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int
+[twitter-img]:  https://img.shields.io/badge/twitter-@sn0int-blue.svg
+[twitter]:      https://twitter.com/sn0int
 [mastodon-img]: https://img.shields.io/badge/mastodon-chaos.social-blue.svg
 [mastodon]:     https://chaos.social/@sn0int
 [registry-img]: https://img.shields.io/website/https/sn0int.com.svg?label=registry
 [registry]:     https://sn0int.com/
 
-sn0int is a semi-automatic OSINT framework and package manager. It was built
-for IT security professionals and bug hunters to gather intelligence about a
-given target or about yourself. sn0int is enumerating attack surface by
-semi-automatically processing public information and mapping the results in a
-unified format for followup investigations.
+sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
+package manager. It's used by IT security professionals, bug bounty hunters,
+law enforcement agencies and in security awareness trainings to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations.
+
+[ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
 
 Among other things, sn0int is currently able to:
 
 - Harvest subdomains from certificate transparency logs and passive dns
+- Mass resolve collected subdomains and scan for http or https services
 - Enrich ip addresses with asn and geoip info
 - Harvest emails from pgp keyservers and whois
 - Discover compromised logins in breaches
 - Find somebody's profiles across the internet
 - Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
-- Harvest data and images from instagram profiles
-- Scan images for nudity
+- Harvest activity and images from social media profiles
+- Basic image processing
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource. None of the investigations listed above are hardcoded
-in the source, instead those are provided by modules that are executed in a
+in the source, instead they are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
-you to ship updates for your modules on your own since you don't need to send a
-pull request.
+you to ship updates for your modules on your own instead of pull-requesting
+them into the sn0int codebase.
 
 For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
@@ -45,6 +49,8 @@ For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https:/
 
 ## Installation
 
+<a href="https://repology.org/project/sn0int/versions"><img align="right" src="https://repology.org/badge/vertical-allrepos/sn0int.svg" alt="Packaging status"></a>
+
 Archlinux
 
     pacman -S sn0int
@@ -53,6 +59,39 @@ Mac OSX
 
     brew install sn0int
 
+Debian/Ubuntu/Kali
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+    apt install debian-keyring
+    gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | apt-key add -
+    apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    echo deb http://apt.vulns.sexy stable main > /etc/apt/sources.list.d/apt-vulns-sexy.list
+    apt update
+    apt install sn0int
+
+Docker
+
+    docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
+
+Alpine
+
+    apk add sn0int
+
+OpenBSD
+
+    pkg_add sn0int
+
+Gentoo
+
+    layman -a pentoo
+    emerge --ask net-analyzer/sn0int
+
+NixOS
+
+    nix-env -i sn0int
+
 For everything else please have a look at the [detailed list][1].
 
 [1]: https://sn0int.readthedocs.io/en/latest/install.html
@@ -62,24 +101,44 @@ For everything else please have a look at the [detailed list][1].
 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
-  - [Debian testing/Debian sid/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-testing-debian-sid-kali)
-  - [Ubuntu/Debian stable](https://sn0int.readthedocs.io/en/latest/install.html#ubuntu-debian-stable)
+  - [Debian &gt;= bullseye, Ubuntu &gt;= 20.04, Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-bullseye-ubuntu-20-04-kali)
+  - [Debian &lt;= buster, Ubuntu &lt;= 19.10](https://sn0int.readthedocs.io/en/latest/install.html#debian-buster-ubuntu-19-10)
+  - [Fedora/CentOS/Redhat](https://sn0int.readthedocs.io/en/latest/install.html#fedora-centos-redhat)
   - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
   - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
+  - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
+  - [NixOS](https://sn0int.readthedocs.io/en/latest/install.html#nixos)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
+- [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
+  - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
+    - [Archlinux](https://sn0int.readthedocs.io/en/latest/build.html#archlinux)
+    - [Mac OSX](https://sn0int.readthedocs.io/en/latest/build.html#mac-osx)
+    - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/build.html#debian-ubuntu-kali)
+    - [Alpine](https://sn0int.readthedocs.io/en/latest/build.html#alpine)
+    - [OpenBSD](https://sn0int.readthedocs.io/en/latest/build.html#openbsd)
+    - [Gentoo](https://sn0int.readthedocs.io/en/latest/build.html#gentoo)
+    - [Windows](https://sn0int.readthedocs.io/en/latest/build.html#windows)
+  - [Building](https://sn0int.readthedocs.io/en/latest/build.html#building)
 - [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
   - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
   - [Adding something to scope](https://sn0int.readthedocs.io/en/latest/usage.html#adding-something-to-scope)
   - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
   - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
   - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
-- [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
-  - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
+- [Autonoscope](https://sn0int.readthedocs.io/en/latest/autonoscope.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/autonoscope.html#domains)
+  - [IPs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#ips)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#urls)
+- [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
+  - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Publish your repo](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-repo)
   - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/database.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/database.html#db-activity)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
 - [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
@@ -94,10 +153,31 @@ For everything else please have a look at the [detailed list][1].
   - [Accounts](https://sn0int.readthedocs.io/en/latest/structs.html#accounts)
   - [Breaches](https://sn0int.readthedocs.io/en/latest/structs.html#breaches)
   - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
+  - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
+  - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
+  - [CryptoAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#cryptoaddrs)
+  - [Activity](https://sn0int.readthedocs.io/en/latest/structs.html#activity)
   - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
-- [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
-- [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
-- [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+    - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
+    - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
+    - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Activity](https://sn0int.readthedocs.io/en/latest/activity.html)
+  - [Anatomy of an event](https://sn0int.readthedocs.io/en/latest/activity.html#anatomy-of-an-event)
+  - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
+  - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
+  - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
+- [Notifications](https://sn0int.readthedocs.io/en/latest/notifications.html)
+  - [Receiving notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#receiving-notifications)
+    - [Telegram](https://sn0int.readthedocs.io/en/latest/notifications.html#telegram)
+    - [Pushover](https://sn0int.readthedocs.io/en/latest/notifications.html#pushover)
+    - [Discord](https://sn0int.readthedocs.io/en/latest/notifications.html#discord)
+    - [Signal](https://sn0int.readthedocs.io/en/latest/notifications.html#signal)
+    - [Writing your own module](https://sn0int.readthedocs.io/en/latest/notifications.html#writing-your-own-module)
+  - [Setting up notification rules](https://sn0int.readthedocs.io/en/latest/notifications.html#setting-up-notification-rules)
+  - [Testing notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#testing-notifications)
+  - [Running sn0int automatically](https://sn0int.readthedocs.io/en/latest/notifications.html#running-sn0int-automatically)
+    - [Monitors](https://sn0int.readthedocs.io/en/latest/notifications.html#monitors)
+    - [Timers](https://sn0int.readthedocs.io/en/latest/notifications.html#timers)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
@@ -113,43 +193,78 @@ For everything else please have a look at the [detailed list][1].
   - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
   - [Diagnosing a sandbox failure](https://sn0int.readthedocs.io/en/latest/sandbox.html#diagnosing-a-sandbox-failure)
 - [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
+  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
+  - [base64_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-decode)
+  - [base64_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-encode)
+  - [base64_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-decode)
+  - [base64_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base64-custom-encode)
+  - [base32_custom_decode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-decode)
+  - [base32_custom_encode](https://sn0int.readthedocs.io/en/latest/reference.html#base32-custom-encode)
   - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
   - [create_blob](https://sn0int.readthedocs.io/en/latest/reference.html#create-blob)
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/reference.html#db-activity)
   - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
   - [error](https://sn0int.readthedocs.io/en/latest/reference.html#error)
-  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
   - [geoip_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#geoip-lookup)
+  - [hex](https://sn0int.readthedocs.io/en/latest/reference.html#hex)
+  - [hmac_md5](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-md5)
+  - [hmac_sha1](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha1)
+  - [hmac_sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-256)
+  - [hmac_sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha2-512)
+  - [hmac_sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-256)
+  - [hmac_sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#hmac-sha3-512)
   - [html_select](https://sn0int.readthedocs.io/en/latest/reference.html#html-select)
   - [html_select_list](https://sn0int.readthedocs.io/en/latest/reference.html#html-select-list)
   - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
   - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
   - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [http_fetch](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch)
+  - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
   - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_ahash](https://sn0int.readthedocs.io/en/latest/reference.html#img-ahash)
+  - [img_dhash](https://sn0int.readthedocs.io/en/latest/reference.html#img-dhash)
+  - [img_phash](https://sn0int.readthedocs.io/en/latest/reference.html#img-phash)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [key_trunc_pad](https://sn0int.readthedocs.io/en/latest/reference.html#key-trunc-pad)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
   - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [mqtt_connect](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-connect)
+  - [mqtt_subscribe](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-subscribe)
+  - [mqtt_recv](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-recv)
+  - [mqtt_ping](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-ping)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [ratelimit_throttle](https://sn0int.readthedocs.io/en/latest/reference.html#ratelimit-throttle)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
+  - [set_err](https://sn0int.readthedocs.io/en/latest/reference.html#set-err)
   - [sha1](https://sn0int.readthedocs.io/en/latest/reference.html#sha1)
   - [sha2_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-256)
   - [sha2_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha2-512)
+  - [sha3_256](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-256)
+  - [sha3_512](https://sn0int.readthedocs.io/en/latest/reference.html#sha3-512)
   - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [sn0int_time](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time)
+  - [sn0int_time_from](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-time-from)
+  - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
+  - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
+  - [sock_options](https://sn0int.readthedocs.io/en/latest/reference.html#sock-options)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
   - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
   - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
@@ -161,8 +276,16 @@ For everything else please have a look at the [detailed list][1].
   - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
   - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [sodium_secretbox_open](https://sn0int.readthedocs.io/en/latest/reference.html#sodium-secretbox-open)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [str_find](https://sn0int.readthedocs.io/en/latest/reference.html#str-find)
+  - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
+  - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
+  - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [strval](https://sn0int.readthedocs.io/en/latest/reference.html#strval)
+  - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
   - [url_escape](https://sn0int.readthedocs.io/en/latest/reference.html#url-escape)
@@ -170,7 +293,19 @@ For everything else please have a look at the [detailed list][1].
   - [url_parse](https://sn0int.readthedocs.io/en/latest/reference.html#url-parse)
   - [url_unescape](https://sn0int.readthedocs.io/en/latest/reference.html#url-unescape)
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
+  - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
+  - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
+  - [ws_connect](https://sn0int.readthedocs.io/en/latest/reference.html#ws-connect)
+  - [ws_options](https://sn0int.readthedocs.io/en/latest/reference.html#ws-options)
+  - [ws_recv_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-text)
+  - [ws_recv_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-binary)
+  - [ws_recv_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-json)
+  - [ws_send_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-text)
+  - [ws_send_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-binary)
+  - [ws_send_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-json)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
+  - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
+  - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)
 
 ## Rationale
 

ci/integration.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import subprocess
 from subprocess import DEVNULL, PIPE
+import shutil
+from pathlib import Path
 import tempfile
 import json
 import sys
@@ -47,10 +49,10 @@ def main(tempdir, binary):
 
     print('[*] installing modules')
     sn0int(tempdir, binary, [
-        'mod install kpcyrd/ctlogs',
-        'mod install kpcyrd/dns-resolve',
-        'mod install kpcyrd/url-scan',
-        'mod install kpcyrd/geoip',
+        'pkg install kpcyrd/ctlogs',
+        'pkg install kpcyrd/dns-resolve',
+        'pkg install kpcyrd/url-scan',
+        'pkg install kpcyrd/geoip',
     ])
 
     print('[*] running ctlogs')
@@ -63,6 +65,7 @@ def main(tempdir, binary):
     print('[*] testing db for subdomains')
     subdomains = sn0int_select(tempdir, binary, ['subdomains'])
     assert {x['value'] for x in subdomains} == {
+        'example.com',
         'www.example.com',
         'm.example.com',
         'dev.example.com',
@@ -91,10 +94,17 @@ def main(tempdir, binary):
     print('[*] testing db for urls')
     urls = sn0int_select(tempdir, binary, ['urls'])
     assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
         ('http://www.example.com/', 200),
         ('https://www.example.com/', 200),
     }
 
+    cache = Path.home() / '.cache' / 'sn0int'
+    if cache.exists():
+        print('[*] copying geoip files')
+        shutil.copytree(cache, tempdir + '/.cache/sn0int', dirs_exist_ok=True)
+
     print('[*] running geoip')
     sn0int(tempdir, binary, [
         'use geoip',

ci/run.sh

@@ -1,41 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    build)
-        cargo build --verbose
-        cargo build --verbose --examples
-        ;;
-    test)
-        ci/run.sh build
-        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
-             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
-        cargo test --verbose
-        cargo test --verbose -- --ignored
-        ;;
-    common)
-        cd sn0int-registry/sn0int-common
-        cargo test --verbose
-        ;;
-    windows)
-        export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
-        ci/run.sh "$2"
-        ;;
-    boxxy)
-        cargo build --verbose --examples
-        if cat ci/boxxy_stage1.txt | RUST_LOG=boxxy cargo run --example boxxy; then
-            echo "SANDOX ERROR: should've crashed"
-            exit 1
-        fi
-        ;;
-    docker)
-        docker build -t sn0int .
-        docker images
-        docker run --rm sn0int --help
-        ;;
-    docker-registry)
-        docker build -t sn0int-registry sn0int-registry/
-        docker images
-        ;;
-esac

ci/setup.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    linux)
-        sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev
-        ;;
-    windows)
-        curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-        7z e sqlite3.zip -y
-        "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
-        ;;
-esac

contrib/docker/Dockerfile.alpine

@@ -1,13 +1,13 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

contrib/docker/Dockerfile.debian

@@ -1,13 +1,13 @@
-FROM rust
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]

contrib/html-toc2md.pl

@@ -24,11 +24,11 @@ while (<$r>) {
 # generate new toc
 while (my $line = <$t>) {
     if ($line =~ /toctree-l(\d).*href="([^"]+)">(.+)<\/a/) {
-        my $space = $1;
+        my $space = int($1)-1;
         my $section = $2;
         my $label = $3;
         $label =~ s/([\[\]])/\\$1/g;
-        print $space==2?"  ":"", "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
+        print " " x ($space*2), "- [$label](https://sn0int.readthedocs.io/en/latest/$section)\n";
     }
 }
 print;

docs/activity.rst

@@ -0,0 +1,166 @@
+Activity
+========
+
+So far we've learned about regular `structs <structs.html>`_, but activity is
+special.
+
+Activity is an event tied to a specific time and topic and has a small amount
+of data piggybacked to it.
+
+Anatomy of an event
+-------------------
+
+``topic``
+    This is some freestyle text used to group events to a specific topic. This
+    must not conflict with other modules unless there's a very good reason.
+
+    The topic should look like ``kpcyrd/example:something``, with ``something``
+    being a meaningful unique identifier for whatever is generating these
+    events, like a mac address or an account name/id.
+
+    The rules around this might become stricter in the future.
+``time``
+    The most important part of the event: The time and date it happened.
+``initial``
+    This value can not be set but might be present in sn0int output. See `Querying events`_.
+``uniq`` (optional)
+    This is an optional feature to deduplicate events. Assuming you're
+    importing posts by an account, you wouldn't want to store a new event for
+    each post you already imported. If you set this field to the technical post
+    id then sn0int would skip the event if it already has an event with the
+    same ``topic`` and ``uniq`` combination to avoid inserting duplicates.
+``latitude`` (optional)
+    Latitude - if you can tie the event to a specific location.
+``longitude`` (optional)
+    Longitude - if you can tie the event to a specific location.
+``radius`` (optional)
+    The location radius in meters. If the position you got has a precision of
+    100 meters set this value to ``100``.
+``content``
+    Arbitrary data that you want to attach to the event. This doesn't need to
+    be a string and can be an arbitrary object that is then stored as json
+    string.
+
+Logging events
+--------------
+
+An ``activity`` event can be logged with ``db_activity``:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Logging an event that has a location attached could look like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        latitude=40.726662,
+        longitude=-74.036677,
+        radius=50,
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Making sure an event is not logged twice can be done with ``uniq``:
+
+.. code-block:: lua
+
+    -- create the first event
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this does nothing because we already have an event with this topic+uniq combination
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this creates a new event because uniq is different
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+    -- this also creates a new event because topic is different
+    db_activity({
+        topic='harness/activity-ping:something-else',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+Querying events
+---------------
+
+There is a commandline interface that can be used to query all events we've
+logged. To get everything (sorted by time)::
+
+    sn0int activity
+
+To limit the output to a specific topic::
+
+    sn0int activity -t harness/activity-ping:dummy
+
+To limit it to a specific time frame::
+
+    # everything since
+    sn0int activity --since 2020-01-13T04:20:00
+    # everything until
+    sn0int activity --until 2020-01-13T04:20:00
+    # both
+    sn0int activity --since yesterday --until today
+
+When using ``--since`` you might also want to know the previous state and use
+it as an initial value. Consider this example::
+
+    2020-01-13 14:30:00 # user goes offline
+    2020-01-13 23:59:00 # user goes online
+    2020-01-14 09:30:00 # user goes idle
+    2020-01-14 14:20:00 # user goes offline
+
+If we're running a query like ``sn0int activity --since 2020-01-14T00:00:00``
+the program consuming the output wouldn't know that the user is initially
+online because we're only getting this data::
+
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+We can tweak this with ``sn0int activity --initial --since
+2020-01-14T00:00:00`` to include one more event that we only use to populate
+the intial state::
+
+    {"id":7,"initial":true,"topic":"foo/bar:asdf","time":"2020-01-13T23:59:00","content":{"state":"online"}}
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+Visualization
+-------------
+
+There is no visualization built in, there may be external frontends for this in
+the future. You're very welcome to write one!

docs/autonoscope.rst

@@ -0,0 +1,81 @@
+Autonoscope
+===========
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+-------
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+---
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+----
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

docs/build.rst

@@ -0,0 +1,83 @@
+Build from source
+=================
+
+It's generally recommended to `install a package <install.html>`_ if available.
+This section is about building the binary from git.
+
+Install dependencies
+--------------------
+
+You need a recent rust compiler. It's usually recommended to install a rust
+compiler with `rustup <https://rustup.rs/>`_, but if you're system ships the
+most recent compiler in a package that works too. Note that some systems aren't
+fully supported by rustup (like OpenBSD and alpine) and you need to install
+rust from a package in that case.
+
+Archlinux
+~~~~~~~~~
+
+.. code-block:: bash
+
+    $ pacman -S geoip2-database libseccomp libsodium publicsuffix-list sqlite
+
+Mac OSX
+~~~~~~~
+
+.. code-block:: bash
+
+    $ brew install libsodium
+
+Debian/Ubuntu/Kali
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix pkg-config
+
+.. warning::
+   On a debian based system make sure you've installed rust with rustup.
+
+Alpine
+~~~~~~
+
+.. code-block:: bash
+
+    $ apk add sqlite-dev libseccomp-dev libsodium-dev
+
+Docker
+~~~~~~
+
+.. code-block:: bash
+
+    $ DOCKER_BUILDKIT=1 docker build -t kpcyrd/sn0int .
+
+OpenBSD
+~~~~~~~
+
+.. code-block:: bash
+
+    $ pkg_add sqlite3 geolite2-city geolite2-asn libsodium
+
+Gentoo
+~~~~~~
+
+.. code-block:: bash
+
+    emerge --ask sys-libs/libseccomp dev-db/sqlite dev-libs/libsodium
+
+Windows
+~~~~~~~
+
+You don't need to install any dependencies on windows, but you need to use a
+different build command in the next section.
+
+Building
+--------
+
+After all dependencies have been installed, simply build the binary:
+
+.. code-block:: bash
+
+    $ cargo build --release
+
+After the build finished the binary is located at ``target/release/sn0int``.

docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'sn0int'
-copyright = '2018, kpcyrd'
+copyright = '2018-2020, kpcyrd'
 author = 'kpcyrd'
 
 # The short X.Y version

docs/database.rst

@@ -46,6 +46,45 @@ triggered and an db_update is performed instead.
    removed from scope with ``noscope``. Everytime you use ``db_add`` you need
    to make sure that the ID that has been returned is not ``nil``.
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add_ttl('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_update
 ---------
 

docs/index.rst

@@ -36,10 +36,14 @@ Getting Started
    :glob:
 
    install
+   build
    usage
+   autonoscope
    scripting
    database
    structs
+   activity
+   notifications
    keyring
    config
    sandbox

docs/install.rst

@@ -1,7 +1,9 @@
 Installation
 ============
 
-If available, please prefer the package shipped by your linux distribution.
+If available, please prefer the package shipped by operating system. If your
+operating system has a package but you're running on older version, please use
+the `build from source <build.html>`_ instructions instead.
 
 Archlinux
 ---------
@@ -17,31 +19,45 @@ Mac OSX
 
     $ brew install sn0int
 
-Debian testing/Debian sid/Kali
-------------------------------
+Debian >= bullseye, Ubuntu >= 20.04, Kali
+-----------------------------------------
 
-Note that debian `doesn't ship the geoip2-database
-<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
-download them automatically during the first run.
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
 
 .. code-block:: bash
 
-    $ apt install build-essential cargo libsqlite3-dev libseccomp-dev publicsuffix
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ sudo apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg kpcyrd@archlinux.org | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ sudo apt update
+    $ sudo apt install sn0int
 
-Ubuntu/Debian stable
+Debian <= buster, Ubuntu <= 19.10
+---------------------------------
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+.. code-block:: bash
+
+    $ sudo apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | sudo apt-key add -
+    $ sudo apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ sudo apt update
+    $ sudo apt install sn0int
+
+Fedora/CentOS/Redhat
 --------------------
 
-cargo in the repos is too old and the build is `going to fail
-<https://github.com/kpcyrd/sn0int/issues/68>`_. You should either install the
-most recent rust version with `rustup <https://rustup.rs/>`_ or use the docker
-instructions instead.
+Using rust+cargo from the repos might work for you, but we only officially
+support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
+at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ dnf install @development-tools libsq3-devel libseccomp-devel libsodium-devel publicsuffix-list
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .
@@ -51,39 +67,47 @@ Docker
 
 .. code-block:: bash
 
-    $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
+    $ docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
 
 Alpine
 ------
 
 .. code-block:: bash
 
-    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ apk add sn0int
 
 OpenBSD
 -------
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f --path .
+    $ pkg_add sn0int
+
+Gentoo
+------
+
+.. code-block:: bash
+
+    $ layman -a pentoo
+    $ emerge --ask net-analyzer/sn0int
+
+NixOS
+-----
+
+.. code-block:: bash
+
+    $ nix-env -i sn0int
 
 Windows
 -------
 
-This is not recommended and only passively maintained. Please prefer linux in a virtual machine if needed.
+This is not recommended and only passively maintained. Please prefer linux in a
+virtual machine if needed.
+
+Make sure rust is installed and setup.
 
 .. code-block:: bash
 
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-    $ 7z e sqlite3.zip -y
-    $ "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
-    $ export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
     $ cargo install -f --path .

docs/keyring.rst

@@ -58,6 +58,10 @@ If the user granted us access to those keys we can read them with ``keyring``:
 This returns a list of all keys in that namespace. Any empty list is returned
 if the user doesn't have any keys in that namespace.
 
+If you want to allow the user to select a specific script you can introduce an
+option that is set by the user and then filter ``creds`` until the
+``access_key`` matches.
+
 Using access keys as source argument
 ------------------------------------
 

docs/notifications.rst

@@ -0,0 +1,311 @@
+Notifications
+=============
+
+If you run sn0int unattended nobody might see the sn0int output. For cases like
+this you can configure notifications to send you a push notification in case
+something interesting happens. This is also especially useful if you have
+sn0int setup to run automatically.
+
+Receiving notifications
+-----------------------
+
+Notifications are just regular sn0int modules. You can install them just like
+any other module or write your own. This section contains walkthroughs on how
+to setup common integrations.
+
+Telegram
+~~~~~~~~
+
+Install the telegram notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-telegram
+
+Open your telegram app and open a chat with ``@botfather``. Send ``/newbot``
+and answer the questions. Copy ``bot_token`` and open this url in your browser:
+
+.. code-block::
+
+    https://api.telegram.org/bot**your_bot_token**/getUpdates
+
+Back on your app, open the t.me link to start a new chat with your bot, then
+send ``/start``. Reload the page in your browser, you should see the new
+message you sent. Copy the ``chat_id``.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-telegram -o bot_token=1337:foobar -o chat_id=1337 'hello world'
+
+You should receive ``hello world`` from your bot on Telegram.
+
+Pushover
+~~~~~~~~
+
+Install the pushover notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-pushover
+
+Signup for pushover and configure the app on your device. Copy th user key
+visible on the pushover dashboard. Click "Create an Application/API Token". Set
+"sn0int" as name and set an icon if you want to. Copy the api token.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-pushover -o user_key=asdf1337 -o api_token=asdf1337 'hello world'
+
+You should receive ``hello world`` as a push notification.
+
+Discord
+~~~~~~~
+
+Install the discord notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-discord
+
+Decide which channel should receive notifications (or create a new one). Open
+the "Server Settings" of your discord server. Click on "Webhooks". Click
+"Create Webhook". Configure the Name and Channel. Copy the Webhook URL.
+
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-discord -o url=https://discord.com/api/webhooks/1337/asdf 'hello world'
+
+You should receive ``hello world`` in your discord channel.
+
+Signal
+~~~~~~
+
+Install the sn0int notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-signal
+
+This module allows end-to-end encrypted notifications, but it's also difficult
+to setup. You need a second phone number and install both `signal-cli
+<https://github.com/AsamK/signal-cli>`_ and `sn0int-signal
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+After you've registered your second phone number with signal-cli, you can use
+sn0int-signal to expose a minimal api for notify-signal. For more detailed
+instructions and how to start the api at boot, see the `sn0int-signal README
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+Read the secret key generated at ``/etc/sn0int-signal.key`` and send a
+notification to the signal phone number:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-signal -o to=+31337 -o secret=asdf 'hello world'
+
+You should receive ``hello world`` from the number signed up with signal-cli.
+
+Writing your own module
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Make sure you've read the detailed instructions on how to get setup with
+`module development <scripting.html>`_.
+
+Create a new sn0int module like this:
+
+.. code-block:: bash
+
+    sn0int new ~/repos/sn0int-modules/notify-custom.lua
+
+Edit the ``-- Source:`` so it takes notifications as input:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        debug(arg)
+        info(arg['subject'])
+        info(arg['body'])
+    end
+
+Execute your script:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom 'hello world'
+
+You most likely need to pass options to avoid hard-coding keys into your
+script. Options can be fetched like this:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        local foo = getopt('foo')
+        if not foo then return 'Missing -o foo= option' end
+
+        info('foo: ' .. foo)
+        info('subject: ' .. arg['subject'])
+    end
+
+And passed like this:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom -o "foo=hello world" 'ohai'
+
+Setting up notification rules
+-----------------------------
+
+We now know how to trigger notifications manually, but we would rather trigger
+notifications if a module runs into something interesting.
+
+You can setup subscriptions on specific topics and then have a notification
+script execute automatically.
+
+Lookup the location of your sn0int config file:
+
+.. code-block:: bash
+
+    sn0int paths
+
+And open it in an editor of your choice:
+
+.. code-block:: bash
+
+    vim /home/user/.config/sn0int.toml
+
+A basic configuration could look like this:
+
+.. code-block:: toml
+
+    # You can have multiple notification sections, this one is named
+    # `demo-telegram-integration`
+    # The label can be set to whatever you want, but you may need to add
+    # double-quotes to use some characters.
+    [notifications.demo-telegram-integration]
+    # If this option is present, the notification must originate from one of
+    # the following workspaces.
+    workspaces = ["default", "some-workspace"]
+    # If this option is present, the notification must match one of the
+    # filters. You can use `*` as a wildcard to match everything except `:`.
+    topics = ["activity:harness/activity-ping:*"]
+    # Mandatory: the module to execute.
+    script = "kpcyrd/notify-telegram"
+    # The options to pass to the module, if any.
+    # Can be accessed with `getopt`
+    options = [
+        "bot_token=1337:foobar",
+        "chat_id=1337",
+    ]
+
+All options except ``script`` are optional, but setting filters is highly
+recommended.
+
+Testing notifications
+---------------------
+
+To test if your configuration works correctly you can create an event manually:
+
+.. code-block:: bash
+
+    sn0int -w some-workspace notify send activity:harness/activity-ping:dummy "hello world"
+
+If it matches any of your rules you should receive a push notifications.
+
+.. note::
+    If you want to test just the routing without actually sending something, add ``--dry-run``.
+
+Running sn0int automatically
+----------------------------
+
+Support for this is going to improve in the future, but you can already set
+this up if you're ok with a slightly buggy experience.
+
+Monitors
+~~~~~~~~
+
+Some modules are long-running and either wait for an event from a server or
+have custom polling built in that's usually configurable with an ``-o
+interval=`` option. If your module has a non-trivial setup phase, an author may
+take this approach.
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-new-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+    Restart=always
+    RestartSec=0
+
+    [Install]
+    WantedBy=multi-user.target
+
+Enable the service to run on boot:
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-new-service.service
+
+Timers
+~~~~~~
+
+If the module is only one-shot you can set it up to run with a timer:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+Setup the timer like this:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.timer
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Timer]
+    OnBootSec=1min
+    OnUnitActiveSec=1h
+
+    [Install]
+    WantedBy=timers.target
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-other-service.timer

docs/reference.rst

@@ -1,6 +1,87 @@
 Function reference
 ==================
 
+asn_lookup
+----------
+
+Run an ASN lookup for a given ip address. The function returns ``asn`` and
+``as_org``. This function may fail.
+
+.. code-block:: lua
+
+    lookup = asn_lookup('1.1.1.1')
+    if last_err() then return end
+
+base64_decode
+-------------
+
+Decode a base64 string with the default alphabet+padding.
+
+.. code-block:: lua
+
+    base64_decode("ww==")
+
+base64_encode
+-------------
+
+Encode a binary array with base64 and the default alphabet+padding.
+
+.. code-block:: lua
+
+    base64_encode("\x00\xff")
+
+base64_custom_decode
+--------------------
+
+Decode a base64 string with custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- base64
+    base64_custom_decode('b2hhaQ==', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '=')
+    -- base64 no padding
+    base64_custom_decode('b2hhaQ', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '')
+    -- base64 url safe
+    base64_custom_decode('b2hhaQ==', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', '=')
+
+base64_custom_encode
+--------------------
+
+Encode a binary array with base64 and custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- base64
+    base64_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '=')
+    -- base64 no padding
+    base64_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '')
+    -- base64 url safe
+    base64_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', '=')
+
+base32_custom_decode
+--------------------
+
+Decode a base32 string with custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- rfc-4648 base32
+    base32_custom_decode('N5UGC2I=', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567', '=')
+    -- z-base-32
+    base32_custom_decode('p7wgn4e', 'ybndrfg8ejkmcpqxot1uwisza345h769', '')
+
+base32_custom_encode
+--------------------
+
+Encode a binary array with base32 and custom alphabet+padding.
+
+.. code-block:: lua
+
+    -- rfc-4648 base32
+    x = base32_custom_encode('ohai', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567', '=')
+    -- z-base-32
+    x = base32_custom_encode('ohai', 'ybndrfg8ejkmcpqxot1uwisza345h769', '')
+
 clear_err
 ---------
 
@@ -35,6 +116,14 @@ for ``DATETIME`` database fields.
 
     now = datetime()
 
+.. note::
+    This format is sn0int specific, to get the current time for scripting use
+    time_unix_ instead.
+
+.. warning::
+    This function is going to be deprecated at some point. Prefer sn0int_time_
+    for new scripts.
+
 db_add
 ------
 
@@ -59,11 +148,33 @@ ttl.
 .. code-block:: lua
 
     -- this link is valid for 2min
-    domain_id = db_add('network-device', {
+    domain_id = db_add_ttl('network-device', {
         network_id=1,
         device_id=13,
     }, 120)
 
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_select
 ---------
 
@@ -135,17 +246,6 @@ Log an error to the terminal.
 
     error('ohai')
 
-asn_lookup
-----------
-
-Run an ASN lookup for a given ip address. The function returns ``asn`` and
-``as_org``. This function may fail.
-
-.. code-block:: lua
-
-    lookup = asn_lookup('1.1.1.1')
-    if last_err() then return end
-
 geoip_lookup
 ------------
 
@@ -166,6 +266,69 @@ This function may fail.
     lookup = geoip_lookup('1.1.1.1')
     if last_err() then return end
 
+hex
+---
+
+Hex encode a list of bytes.
+
+.. code-block:: lua
+
+    hex("\x6F\x68\x61\x69\x0A\x00")
+
+hmac_md5
+--------
+
+Calculate an hmac with md5. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_md5("secret", "my authenticated message")
+
+hmac_sha1
+---------
+
+Calculate an hmac with sha1. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha1("secret", "my authenticated message")
+
+hmac_sha2_256
+-------------
+
+Calculate an hmac with sha2_256. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha2_256("secret", "my authenticated message")
+
+hmac_sha2_512
+-------------
+
+Calculate an hmac with sha2_512. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha2_512("secret", "my authenticated message")
+
+hmac_sha3_256
+-------------
+
+Calculate an hmac with sha3_256. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha3_256("secret", "my authenticated message")
+
+hmac_sha3_512
+-------------
+
+Calculate an hmac with sha3_512. Returns a binary array.
+
+.. code-block:: lua
+
+    hmac_sha3_512("secret", "my authenticated message")
+
 html_select
 -----------
 
@@ -220,11 +383,20 @@ options are set. The following options are available:
   The request body that should be json encoded.
 ``form``
   The request body that should be form encoded.
+``follow_redirects``
+  Automatically follow redirects, up to the specified number. If set to 1, only
+  one redirect is going to be followed. Defaults to 0 so redirects aren't
+  followed.
 ``body``
   The raw request body as string.
 ``into_blob``
   If true, the response body is stored in blob storage and a blob reference is
   returned as ``blob`` instead of the full body.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``binary``
+  Set to ``true`` to get the http response as raw bytes.
 
 This function may fail.
 
@@ -238,7 +410,7 @@ This function may fail.
     })
     resp = http_send(req)
     if last_err() then return end
-    if resp["status"] ~= 200 then return "invalid status code" end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
 
 http_send
 ---------
@@ -252,6 +424,8 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``binary``
+  The response body as bytes (if ``binary=true``)
 ``blob``
   If ``into_blob`` was enabled for the request the body is downloaded into blob
   storage with a reference to the body in this field.
@@ -266,7 +440,46 @@ the following keys:
     })
     resp = http_send(req)
     if last_err() then return end
-    if resp["status"] ~= 200 then return "invalid status code" end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+
+http_fetch
+----------
+
+This does an http_send_ and also automatically validate the status code.
+
+.. note::
+   You almost always want this when setting the ``into_blob`` option since this
+   function validates the status code *before* inserting the response body into
+   blob storage.
+
+.. code-block:: lua
+
+    -- short form
+    data = http_fetch(req)
+    if last_err() then return end
+
+    -- long form
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+
+http_fetch_json
+---------------
+
+Identical to http_fetch_ but also automatically parses the response body as json.
+
+.. code-block:: lua
+
+    -- short form
+    data = http_fetch_json(req)
+    if last_err() then return end
+
+    -- long form
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http status error: ' .. resp['status'] end
+    data = json_decode(resp['text'])
+    if last_err() then return end
 
 img_load
 --------
@@ -291,6 +504,39 @@ Extract exif metadata from an image.
     if last_err() then return end
     debug(exif)
 
+img_ahash
+---------
+
+Calculate the Mean (aHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_ahash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_dhash
+---------
+
+Calculate the Gradient (dHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_dhash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_phash
+---------
+
+Calculate the DCT (pHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_phash(blob)
+    if last_err() then return end
+    debug(hash)
+
 img_nudity
 ----------
 
@@ -312,6 +558,13 @@ Log an info to the terminal.
 
     info('ohai')
 
+intval
+------
+
+Parse a number from a string.
+
+    x = strval('1234')
+
 json_decode
 -----------
 
@@ -346,6 +599,17 @@ Encode a datastructure into a string.
     })
     print(x)
 
+key_trunc_pad
+-------------
+
+Truncate/pad a key to a given length.
+
+.. code-block:: lua
+
+    -- if longer than 32 bytes: truncate to 32
+    -- if shorter than 32 bytes: pad with \x00
+    local key = key_trunc_pad(password, 32, 0)
+
 keyring
 -------
 
@@ -380,6 +644,64 @@ Hash a byte array with md5 and return the results as bytes.
 
     hex(md5("\x00\xff"))
 
+mqtt_connect
+------------
+
+Connect to an mqtt broker.
+
+.. code-block:: lua
+
+    local sock = mqtt_connect('mqtts://mqtt.example.com', {
+        username='foo',
+        password='secret',
+    })
+    if last_err() then return end
+
+mqtt_subscribe
+--------------
+
+Subscribe to a topic. Right now only QoS 0 is supported.
+
+.. code-block:: lua
+
+    mqtt_subscribe(sock, '#', 0)
+    if last_err() then return end
+
+mqtt_recv
+---------
+
+Receive an mqtt packet. This is not necessarily a publish packet and more
+packets might be added in the future, so you need to check the type
+specifically.
+
+If a read timeout has been set with mqtt_connect_ this function returns ``nil``
+in case of a read timeout.
+
+.. code-block:: lua
+
+    local pkt = mqtt_recv(sock)
+    if last_err() then return end
+    if pkt == nil then
+        -- read timeout, consider sending a ping or disconnect if the previous ping failed
+    elseif pkt['type'] == 'pong' then
+        -- broker sent a pong
+    elseif pkt['type'] == 'publish' then
+        local payload = utf8_decode(pkt['body'])
+        if last_err() then return end
+        info(payload)
+    end
+
+mqtt_ping
+---------
+
+Send a pingreq packet, causing the broker to send a pingresp. This is used to
+make sure the connection is still working correctly.
+
+.. code-block:: lua
+
+    mqtt_ping(sock)
+    if last_err() then return end
+
 pgp_pubkey
 ----------
 
@@ -388,7 +710,8 @@ Same as pgp_pubkey_armored_, but without the unarmor step.
 pgp_pubkey_armored
 ------------------
 
-Extract uids out of a rfc 4880 pgp public key. This function may fail.
+Extract ``uids``, ``sigs`` and the ``fingerprint`` out of an rfc 4880 pgp
+public key. This function may fail.
 
 .. code-block:: lua
 
@@ -459,6 +782,22 @@ Returns the parent domain according to the public suffix list. For
     domain = psl_domain_from_dns_name('www.a.b.c.d.example.co.uk')
     print(domain == 'example.co.uk')
 
+ratelimit_throttle
+------------------
+
+Create a ratelimit that can only be passed x times every y milliseconds. This
+limit is global for a single ``run`` and also works with threads.
+
+.. code-block:: lua
+
+    -- allow this to pass every 250ms
+    ratelimit_throttle('foo', 1, 250)
+    -- allow this to pass not more than 4 times per second
+    ratelimit_throttle('foo', 4, 1000)
+
+This is useful if you need to coordinate your executions to stay below a
+certain request threshold.
+
 regex_find
 ----------
 
@@ -492,6 +831,41 @@ Same as regex_find_, but returns all matches.
     print(m[3][1] == 'ef')
     print(m[3][2] == 'f')
 
+semver_match
+------------
+
+Compare a version to a version requirement. This can be used with
+sn0int_version_ to test for certain features or behavior.
+
+.. code-block:: lua
+
+    semver_match('=0.11.2', sn0int_version())
+    semver_match('>0.11.2', sn0int_version())
+    semver_match('<0.11.2', sn0int_version())
+    semver_match('~0.11.2', sn0int_version())
+    semver_match('^0.11.2', sn0int_version())
+    semver_match('0.11.2', sn0int_version()) -- synonym for ^0.11.2
+    semver_match('<=0.11.2', sn0int_version())
+    semver_match('>=0.11.2', sn0int_version())
+    semver_match('>=0.4.0, <=0.10.0', sn0int_version())
+
+set_err
+-------
+
+Manipulate the global error object. If you want to exit the main ``run``
+function with an error you can simply return a string, but those are difficult
+to propagate through functions. ``set_err`` specifically assigns an error to
+the global error object that are also used by all other rust functions.
+
+.. code-block:: lua
+
+    function foo()
+        set_err("something failed")
+    end
+
+    foo()
+    if last_err() then return end
+
 sha1
 ----
 
@@ -519,6 +893,24 @@ Hash a byte array with sha2_512 and return the results as bytes.
 
     hex(sha2_512("\x00\xff"))
 
+sha3_256
+--------
+
+Hash a byte array with sha3_256 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha3_256("\x00\xff"))
+
+sha3_512
+--------
+
+Hash a byte array with sha3_512 and return the results as bytes.
+
+.. code-block:: lua
+
+    hex(sha3_512("\x00\xff"))
+
 sleep
 -----
 
@@ -529,14 +921,112 @@ only used for debugging.
 
     sleep(1)
 
+sn0int_time
+-----------
+
+Return current time in UTC. This function is suitable to determine datetimes
+for ``DATETIME`` database fields.
+
+.. code-block:: lua
+
+    now = sn0int_time()
+
+.. note::
+    This format is sn0int specific, to get the current time for scripting use
+    time_unix_ instead.
+
+sn0int_time_from
+----------------
+
+Identical to sn0int_time_ but uses a unix timestamp in seconds instead of the
+current time. This function is compatible with time_unix_ and strptime_.
+
+.. code-block:: lua
+
+    time = sn0int_time_from(1567931337)
+
+sn0int_version
+--------------
+
+Get the current sn0int version string. This can be used with semver_match_ to
+test for certain features or behavior.
+
+.. code-block:: lua
+
+    info(sn0int_version())
+
 sock_connect
 ------------
 
 Create a tcp connection.
 
+The following options are available:
+
+``tls``
+  Set to true to enable tls (certificates are validated)
+``sni_value``
+  Instead of the host argument, use a custom string for the sni extension.
+``disable_tls_verify``
+  **Danger**: disable tls verification. This disables all security on the
+  connection. Note that sn0int is still rather strict, you're going to run into
+  issues if you need support for insecure ciphers.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
 .. code-block:: lua
 
-    sock = sock_connect("127.0.0.1", 1337)
+    sock = sock_connect("127.0.0.1", 1337, {
+        tls=true,
+    })
+
+sock_upgrade_tls
+----------------
+
+Take an existing tcp connection and start a tls handshake. The options are the
+same as sock_connect_ but the ``tls`` value is always assumed to be true.
+
+The sni value needs to be set specifically, otherwise the sni extension is
+disabled.
+
+Using this function specifically returns some extra information that is
+discarded when using sock_connect_ directly with ``tls=true``.
+
+.. code-block:: lua
+
+    sock = sock_connect("127.0.0.1", 1337, {})
+    if last_err() then return end
+
+    tls = sock_upgrade_tls(sock, {
+        sni_value='example.com',
+    })
+    if last_err() then return end
+
+    info(tls)
+
+sock_options
+------------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock_options(sock, {
+        read_timeout=3,
+    })
 
 sock_send
 ---------
@@ -639,6 +1129,27 @@ Overwrite the default ``\n`` newline.
 
     sock_newline(sock, "\r\n")
 
+sodium_secretbox_open
+---------------------
+
+Use authenticated symetric crypto to decrypt a given message.
+
+Internally this is ``crypto_secretbox_xsalsa20poly1305``.
+
+The key **must** be 32 bytes, see key_trunc_pad_ if necessary.
+
+The first 24 bytes of the encrypted message are expected to be the nonce.
+
+.. code-block:: lua
+
+    plain = sodium_secretbox_open(encrypted, key)
+    if last_err() then return end
+
+    txt = utf8_decode(plain)
+    if last_err() then return end
+
+    info(txt)
+
 status
 ------
 
@@ -660,6 +1171,83 @@ Read a line from stdin. The final newline is not removed.
 .. note::
    This only works with `sn0int run --stdin`.
 
+.. TODO: add stdin_read_line and deprecate stdin_readline
+
+stdin_read_to_end
+-----------------
+
+Read stdin until EOF as a utf-8 string.
+
+.. code-block:: lua
+
+    stdin_read_to_end()
+
+.. note::
+   This only works with `sn0int run --stdin`.
+
+str_find
+--------
+
+Returns the byte index of the first character that matches the pattern. This is
+explicitly a literal match instead of a lua pattern.
+
+If no match is found, returns ``nil``.
+
+.. code-block:: lua
+
+    x = str_find('asdf', 'sd')
+    print(x == 2)
+
+str_replace
+-----------
+
+Replaces all matches of a pattern in a string. This is explicitly a literal
+match instead of a lua pattern.
+
+If no match is found, an unmodified copy is returned.
+
+.. code-block:: lua
+
+    x = str_replace('this is old', 'old', 'new')
+    print(x == 'this is new')
+
+strftime
+--------
+
+Format a timestamp generated with time_unix_ into a date, see `strftime rules`_.
+
+.. code-block:: lua
+
+    t = strftime('%d/%m/%Y %H:%M', 1558584994)
+
+strptime
+--------
+
+Parse a date into a unix timestamp, see `strftime rules`_.
+
+.. code-block:: lua
+
+    t = strptime('%d/%m/%Y %H:%M', '23/05/2019 04:16')
+
+.. _strftime rules: https://docs.rs/chrono/0.4.6/chrono/format/strftime/index.html
+
+strval
+------
+
+Convert a number into a string.
+
+    x = strval(1234)
+
+time_unix
+---------
+
+Get the current time as seconds since ``January 1, 1970 0:00:00 UTC``, also
+known as UNIX timestamp. This timestamp can be formated using strftime_.
+
+.. code-block:: lua
+
+    now = time_unix()
+
 url_decode
 ----------
 
@@ -749,6 +1337,132 @@ Decodes a list of bytes/numbers into a string. This function might fail.
     if last_err() then return end
     print(x == 'AAAA')
 
+warn
+----
+
+Log a warning to the terminal.
+
+.. code-block:: lua
+
+    warn('ohai')
+
+warn_once
+---------
+
+Log a warning to the terminal once. This can be used to print a warning to the
+user without printing the same warning for each struct we're processing during
+a ``run`` execution.
+
+.. code-block:: lua
+
+    warn_once('ohai')
+    warn_once('ohai')
+
+ws_connect
+----------
+
+Create a websocket connection. The url format is ``ws://example.com/asdf``,
+``wss://`` is also supported.
+
+The following options are available:
+
+``headers``
+  A map of additional headers that should be set for the request.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock = ws_connect("wss://example.com/asdf", {})
+
+ws_options
+----------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    ws_options(sock, {
+        read_timeout=3,
+    })
+
+ws_recv_text
+------------
+
+Wait until the server sends a text frame. A binary frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_text(sock)
+
+ws_recv_binary
+--------------
+
+Wait until the server sends a binary frame. A text frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_binary(sock)
+
+ws_recv_json
+------------
+
+Identical to ws_send_text_ but automatically runs json_decode_ on the
+response.
+
+.. code-block:: lua
+
+    msg = ws_recv_json(sock)
+
+ws_send_text
+------------
+
+Send a text frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, "ohai!")
+
+ws_send_binary
+--------------
+
+Send a binary frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_binary(sock, "\x00\x01\x02")
+
+ws_send_json
+------------
+
+Encode the object as json string and send it as a text frame on the websocket
+connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, {
+        foo="ohai!",
+        x={
+            y={1,3,3,7},
+        },
+    })
+
 x509_parse_pem
 --------------
 
@@ -783,3 +1497,35 @@ Parse a pem encoded certificate. This function might fail.
     ]])
     if last_err() then return end
     print(x)
+
+xml_decode
+----------
+
+Decode a lua value from an xml document.
+
+.. code-block:: lua
+
+    x = xml_decode('<body><foo fizz="buzz">bar</foo></body>')
+    if last_err() then return end
+
+    body = x['children'][1]
+    foo = body['children'][1]
+
+    print(foo['attrs']['fizz'])
+    print(foo['text'])
+
+xml_named
+---------
+
+Get a named child element from a parent element.
+
+.. code-block:: lua
+
+    x = xml_decode('<body><foo fizz="buzz">bar</foo></body>')
+    if last_err() then return end
+
+    body = x['children'][1]
+    foo = xml_named(body, 'foo')
+    if foo ~= nil then
+        print(foo)
+    end

docs/sandbox.rst

@@ -6,7 +6,7 @@ child process. It's important to note that there's a basic sandbox that's
 active on every operating system, and there's a second line of defense on
 supported operating systems.
 
-The first line of defense is the restrictive stdlib. It's assumed that and
+The first line of defense is the restrictive stdlib. It's assumed that an
 attacker gains full control over the lua code and is able to call any function
 with arbitrary arguments. The stdlib only provides functions that are
 considered safe, so for example it's not possible to start a process or open a
@@ -119,7 +119,7 @@ You might experience a sandbox failure, especially on architectures that are
 less popular. This usually looks like this::
 
     [sn0int][example][kpcyrd/ctlogs] > run
-    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [-] Failed "example.com": Sandbox child has crashed
     [+] Finished kpcyrd/ctlogs (1 errors)
 
 A module that never finishes could also mean an IO thread inside the worker got

docs/scripting.rst

@@ -1,11 +1,11 @@
-Scripting
-=========
+Writing your first module
+=========================
 
 Scripting is the core feature in sn0int. It's not strictly required, but if you
 want to write your own modules, this section is for you.
 
-Write your first module
------------------------
+Creating a repository
+---------------------
 
 It's highly recommended to use a VCS for development, so let's start by setting
 that up. We're going to assume you store your repos in ``~/repos`` but you're
@@ -14,6 +14,12 @@ free to change that to something else::
     $ git init ~/repos/sn0int-modules
     $ cd ~/repos/sn0int-modules
 
+.. note::
+   If you're using github you can also create a repo from the `module repo
+   template`_.
+
+.. _module repo template: https://github.com/sn0int/sn0int-modules
+
 We need to add this folder to the sn0int config file so it's correctly detected
 when starting sn0int. Open the `config file <config.html>`_ in your prefered
 editor. Note that the file does not exist by default and the path is different
@@ -22,7 +28,7 @@ with::
 
     $ vim ~/.config/sn0int.toml
 
-Add the follwing::
+Add the following::
 
     [namespaces]
     your_github_name = "~/repos/sn0int-modules"
@@ -90,21 +96,26 @@ database.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
-        print(subdomain)
+        info(subdomain)
     end
 
-Combined with the header we wrote previously we can already execute this
-module. Make sure you've added a domain to scope with ``add domain
-example.com``, save your file and run it like this::
+This is already enough to execute it. Make sure you've added a domain to scope
+with ``add domain example.com``, save your file and run it like this::
 
     sn0int run -f ./first.lua
 
-We should see some output by our print function.
+We should see some output by our info function.
 
 .. note::
-   ``print`` is useful for development but must be removed before publishing.
+   ``info`` is useful for development but you usually want your module to run
+   quietly, so before publishing either remove it or replace it with ``debug``.
 
 Next, we want to actually resolve that name, we're going to use the ``dns``
 function for that. This function takes a name and a query type and returns a
@@ -114,6 +125,11 @@ truth-y.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
@@ -122,16 +138,22 @@ truth-y.
         })
         if last_err() then return end
 
-        print(records)
+        info(records)
     end
 
 If you run your module again you're going to see some output, either
 ``{"answers":[somedata],"error":null}`` or
-``{"answers":[],"error":"NXDomain"}``. We decide that we add the subdomain to
-our scope and set it to resolvable if ``error`` is ``nil``.
+``{"answers":[],"error":"NXDomain"}``. If the dns reply doesn't indicate an
+error this means the subdomain exists and we can add it to our database with
+``resolvable`` being set to ``true``.
 
 .. code-block:: lua
 
+    -- Description: Scan for www. subdomains
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
@@ -152,41 +174,20 @@ our scope and set it to resolvable if ``error`` is ``nil``.
 .. hint::
    See the database section to understand how the database works in detail.
 
-If we execute our module one more time it's going to log that it discovered a
-subdomain, if it doesn't, try adding more domains to scope. Note that this only
-happens the first time. Modules that don't discover anything or don't discover
-anything new exit silently.
-
-After putting everything together, our final module looks like this:
-
-.. code-block:: lua
-
-    -- Description: ohai wurld
-    -- Version: 0.1.0
-    -- Source: domains
-    -- License: GPL-3.0
-
-    function run(arg)
-        subdomain = 'www.' .. arg['value']
-
-        records = dns(subdomain, {
-            record='A'
-        })
-        if last_err() then return end
-
-        if records['success'] ~= nil then
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=subdomain,
-                resolvable=true,
-            })
-        end
-    end
+If we execute our finished module one more time it's going to log that it
+discovered a subdomain, if it doesn't, try adding more domains to scope. Note
+that this only happens the first time. Modules that don't discover anything or
+don't discover anything new exit silently.
 
 There's still some room for improvement, for example, since we already resolved
 that record, we could also add the ip address to the scope and link it to the
 subdomain we added.
 
+.. hint::
+   For debugging purposes you can increase the verbosity with ``sn0int run -v``
+   so database operations are logged even if nothing was changed, or with
+   ``sn0int run -vv`` to enable ``debug()`` output.
+
 Publish your module
 -------------------
 
@@ -205,6 +206,24 @@ Afterwards publish your module with::
 
     sn0int publish ./first.lua
 
+Please also make sure you publish your repository to github so other people can
+submit pull requests. The recommended repository location is::
+
+    https://github.com/<your-username>/sn0int-modules
+
+Publish your repo
+-----------------
+
+It is highly recommended to publish your repository on github so people can
+file issues and pull requests for your module. If you've been following along
+with the github template you can simply commit your changes and push them.
+
+Your repository would look like one of these:
+
+- https://github.com/kpcyrd/sn0int-modules
+- https://github.com/ysf/sn0int-modules
+- https://github.com/cybiere/sn0int-modules
+
 Reading data from stdin
 -----------------------
 

docs/structs.rst

@@ -145,6 +145,8 @@ connected to.
     Latitude of the networks location.
 ``longitude``
     Longitude of the networks location.
+``description``
+    A human readable description in case the value is a technical identifier.
 
 Accounts
 --------
@@ -166,6 +168,12 @@ A users account or profile on a webservice, like github or instagram.
     The url of the public profile if available.
 ``last_seen``
     The last time this account has been active/online.
+``birthday``
+    The users birthday set on the account.
+``phonenumber``
+    The phonenumber associated with the account.
+``profile_pic``
+    The blob identifier of the users current profile picture.
 
 Breaches
 --------
@@ -207,6 +215,79 @@ Images
 ``phash``
     The DCT (pHash) perceptual hash.
 
+Ports
+-----
+
+The status of a port on an ip address.
+
+``ip_addr_id``
+    The numeric id of an ipaddr struct.
+``ip_addr``
+    The actual ipaddr.
+``port``
+    The port number.
+``status``
+    The status of the port, either ``open`` or ``closed``.
+``banner``
+    The service banner we discovered on this port.
+``service``
+    The service that is running on this port.
+``version``
+    The version of the service running on this port.
+
+Netblocks
+---------
+
+A netblock is a network address range that has been allocated to an individual,
+organization or company. Those are commonly found when running whois lookups on
+an ip address.
+
+Consider the following example: Running a whois lookup on ``140.82.118.4`` (one
+of the addresses currently in use by github) returns that this address belongs
+to the netrange ``140.82.112.0 - 140.82.127.255``, so the netblock in this case
+is ``140.82.112.0/20``.
+
+``family``
+    This is either ``4`` or ``6`` and populated automatically.
+``value``
+    This is the network range in CIDR notation.
+``asn``
+    The number of the autonomous system this network belongs to.
+``as_org``
+    The organization of the autonomous system this network belongs to.
+``description``
+    This field isn't strictly defined and meant to be used as a human
+    meaningful name if available.
+
+CryptoAddrs
+-----------
+
+A cryptoaddr is any cryptocurrency address and not tied to a specific currency.
+
+``value``
+    The address string. This looks like ``1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2``.
+``currency``
+    The identifier for a specific currency. This is usually the ticker symbols,
+    like ``xbt``, ``zec`` or ``xmr``.
+``denominator``
+    Balance is tracked internally using 64 bit integers (signed, for technical reasons). Balance is supposed to be the lowest unit, so in case of bitcoin you'd write ``100,000,000`` satoshi instead of ``1`` bitcoin. Since this value is inconvinient to work with we're using the denominator to display values. In case of bitcoin you'd set it to ``8``.
+``balance``
+    The current balance of the address, in the lowest possible unit. In case of bitcoin this would be satoshis.
+``received``
+    The total amount of currency received by this address.
+``first_seen``
+    The first time currency was sent to this address.
+``last_withdrawal``
+    The last time a transaction signed by this address was observed.
+``description``
+    A human readable note for this address.
+
+Activity
+--------
+
+Activity is different from all other structs, have a look at the `Activity
+Section <activity.html>`_.
+
 Relations
 ---------
 

docs/usage.rst

@@ -27,12 +27,12 @@ number of recommended modules::
     [+] Downloading "GeoLite2-City.mmdb"
     [+] Downloading "GeoLite2-ASN.mmdb"
     [+] Loaded 0 modules
-    [*] No modules found, run quickstart to install default modules
+    [*] No modules found, run pkg quickstart to install default modules
     [sn0int][default] >
 
-Typing ``quickstart`` is going to get you a fair number of featured modules::
+Typing ``pkg quickstart`` is going to get you a fair number of featured modules::
 
-    [sn0int][default] > quickstart
+    [sn0int][default] > pkg quickstart
     [+] Installing kpcyrd/asn
     [+] Installing kpcyrd/ctlogs
     [+] Installing kpcyrd/dns-resolve
@@ -105,7 +105,7 @@ Running a module
 Now that we have something to get started with, we can run our first module.
 First lets list all modules we have::
 
-    [sn0int][demo] > mod list
+    [sn0int][demo] > pkg list
     kpcyrd/asn (0.1.0)
         Run a asn lookup for an ip address
     kpcyrd/ctlogs (0.1.0)
@@ -149,7 +149,7 @@ some of them in a browser but hold on, there's a more efficient way to approach
 this.
 
 .. hint::
-   You can run the modules concurrently with ``run -j 8``.
+   You can run the modules concurrently with ``run -j3``.
 
 Running followup modules on the results
 ---------------------------------------

examples/maxmind.rs

@@ -1,61 +1,27 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
 use sn0int::paths;
-use std::fs;
 use std::net::IpAddr;
+use std::path::Path;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
 pub enum Args {
-    #[structopt(name="dl")]
-    Download(Download),
     #[structopt(name="asn")]
     Asn(AsnArgs),
     #[structopt(name="geoip")]
     GeoIP(GeoIPArgs),
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Download {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-impl Download {
-    fn run(&self) -> Result<()> {
-        let path = paths::cache_dir()?.join(&self.target);
-        if self.extract_only {
-            let body = fs::read(&self.url)?;
-            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
-        } else {
-            GeoIP::download(path, &self.filter, &self.url)?;
-        }
-        Ok(())
-    }
-}
-
 #[derive(Debug, StructOpt)]
 pub struct AsnArgs {
     ip: IpAddr,
 }
 
 impl AsnArgs {
-    fn run(&self) -> Result<()> {
-        let asndb = AsnDB::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = AsnDB::cache_path(cache_dir)?;
+        let asndb = AsnDB::open(&path)?;
 
         let asn = asndb.lookup(self.ip)?;
         println!("{:#?}", asn);
@@ -70,8 +36,9 @@ pub struct GeoIPArgs {
 }
 
 impl GeoIPArgs {
-    fn run(&self) -> Result<()> {
-        let geoip = GeoIP::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = GeoIP::cache_path(cache_dir)?;
+        let geoip = GeoIP::open(&path)?;
 
         let lookup = geoip.lookup(self.ip)?;
         println!("{:#?}", lookup);
@@ -84,10 +51,10 @@ impl GeoIPArgs {
 fn run() -> Result<()> {
     let args = Args::from_args();
     debug!("{:?}", args);
+    let cache_dir = paths::cache_dir()?;
     match args {
-        Args::Download(args) => args.run(),
-        Args::Asn(args) => args.run(),
-        Args::GeoIP(args) => args.run(),
+        Args::Asn(args) => args.run(&cache_dir),
+        Args::GeoIP(args) => args.run(&cache_dir),
     }
 }
 

migrations/2019-05-30-142142_ports/down.sql

@@ -0,0 +1 @@
+DROP TABLE ports;

migrations/2019-05-30-142142_ports/up.sql

@@ -0,0 +1,17 @@
+CREATE TABLE ports (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);

migrations/2019-07-04-233515_autonoscope/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE autonoscope;

migrations/2019-07-04-233515_autonoscope/up.sql

@@ -0,0 +1,8 @@
+-- Your SQL goes here
+CREATE TABLE autonoscope (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    object VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    scoped BOOLEAN NOT NULL,
+    CONSTRAINT autonoscope_unique UNIQUE (object, value)
+);

migrations/2019-07-27-152215_netblocks/down.sql

@@ -0,0 +1 @@
+DROP TABLE netblocks;

migrations/2019-07-27-152215_netblocks/up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE netblocks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    asn INTEGER,
+    as_org VARCHAR,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2019-08-11-073709_misc-fields/down.sql

@@ -0,0 +1,41 @@
+PRAGMA foreign_keys=off;
+
+-- accounts
+CREATE TABLE _accounts_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    service VARCHAR NOT NULL,
+    username VARCHAR NOT NULL,
+    displayname VARCHAR,
+    email VARCHAR,
+    url VARCHAR,
+    last_seen DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    CONSTRAINT account_unique UNIQUE (value)
+);
+
+INSERT INTO _accounts_new (id, value, service, username, displayname, email, url, last_seen, unscoped)
+    SELECT id, value, service, username, displayname, email, url, last_seen, unscoped
+    FROM accounts;
+
+DROP TABLE accounts;
+ALTER TABLE _accounts_new RENAME TO accounts;
+
+-- networks
+CREATE TABLE _networks_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+INSERT INTO _networks_new (id, value, unscoped, latitude, longitude)
+    SELECT id, value, unscoped, latitude, longitude
+    FROM networks;
+
+DROP TABLE networks;
+ALTER TABLE _networks_new RENAME TO networks;
+
+PRAGMA foreign_keys=on;

migrations/2019-08-11-073709_misc-fields/up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE accounts ADD COLUMN phonenumber VARCHAR;
+ALTER TABLE accounts ADD COLUMN profile_pic VARCHAR;
+ALTER TABLE accounts ADD COLUMN birthday VARCHAR;
+ALTER TABLE networks ADD COLUMN description VARCHAR;

migrations/2019-11-19-154056_cryptoaddr/down.sql

@@ -0,0 +1 @@
+DROP TABLE cryptoaddrs;

migrations/2019-11-19-154056_cryptoaddr/up.sql

@@ -0,0 +1,13 @@
+CREATE TABLE cryptoaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    currency VARCHAR,
+    denominator INTEGER,
+    balance BIGINT,
+    received BIGINT,
+    first_seen DATETIME,
+    last_withdrawal DATETIME,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    description VARCHAR,
+    CONSTRAINT netblock_unique UNIQUE (value)
+);

migrations/2020-01-09-024234_activity/down.sql

@@ -0,0 +1,31 @@
+DROP TABLE activity;
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-01-09-024234_activity/up.sql

@@ -0,0 +1,44 @@
+CREATE TABLE activity (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    topic VARCHAR NOT NULL,
+    time DATETIME NOT NULL,
+    uniq VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    radius INTEGER,
+    content VARCHAR NOT NULL
+);
+CREATE UNIQUE INDEX activity_uniq ON activity(topic, uniq);
+CREATE INDEX activity_topic ON activity(topic);
+CREATE INDEX activity_time ON activity(time);
+CREATE INDEX activity_topic_time ON activity(topic, time);
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, expire)
+    SELECT id, family, key, expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/up.sql

@@ -0,0 +1,19 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, value, expire)
+    SELECT id, family, key, "unknown", expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

modules/dev/arp-scan.lua

@@ -1,45 +0,0 @@
--- Description: Parse arp-scan output
--- Version: 0.3.0
--- License: GPL-3.0
-
--- sudo arp-scan -qglI wlp3s0
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        m = regex_find('(.+)\t(.+)', x)
-        if m ~= nil then
-            ipaddr = m[2]
-            mac = m[3]
-            now = datetime()
-
-            device_id = db_add('device', {
-                value=mac,
-                last_seen=now,
-            })
-            if last_err() then return end
-
-            db_add_ttl('network-device', {
-                network_id=network_id,
-                device_id=device_id,
-                ipaddr=ipaddr,
-                last_seen=now,
-            }, 300)
-            if last_err() then return end
-        end
-    end
-end

modules/dev/asn.lua

@@ -1,16 +0,0 @@
--- Description: Run a asn lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = asn_lookup(arg['value'])
-    if last_err() then return end
-
-    if arg['asn'] ~= lookup['asn'] or arg['as_org'] ~= lookup['as_org'] then
-        db_update('ipaddr', arg, {
-            asn=lookup['asn'],
-            as_org=lookup['as_org'],
-        })
-    end
-end

modules/dev/axfr.lua

@@ -1,145 +0,0 @@
--- Description: Try a zone transfer for subdomains
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function strip_root_dot(name)
-    local m = regex_find("(.+)\\.$", name)
-    if last_err() then return end
-
-    if m == nil then
-        return name
-    else
-        return m[2]
-    end
-end
-
-function add_pointer(name)
-    -- select psl+1
-    local domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    local subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-end
-
-function iter_axfr(zone, arg)
-    debug(arg)
-
-    local name = arg[1]
-    local r = arg[2]
-
-    -- select psl+1
-    local domain = psl_domain_from_dns_name(name)
-    if last_err() then return end
-
-    -- add domain
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    -- add subdomain
-    local subdomain_id = db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-    if last_err() then return end
-
-    -- this is a A record
-    if r['A'] ~= nil then
-        -- add the name and ip
-        local ipaddr_id = db_add('ipaddr', {
-            family='4',
-            value=r['A'],
-        })
-        if last_err() then return end
-
-        db_add('subdomain-ipaddr', {
-            subdomain_id=subdomain_id,
-            ip_addr_id=ipaddr_id,
-        })
-        if last_err() then return end
-    end
-
-    if r['CNAME'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['CNAME'])
-        add_pointer(name)
-    end
-
-    if r['NS'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['NS'])
-        add_pointer(name)
-    end
-
-    if r['MX'] ~= nil then
-        -- add the name and the name it's pointing to
-        name = strip_root_dot(r['MX'][2])
-        add_pointer(name:lower())
-    end
-end
-
-function iter_a(zone, arg)
-    if arg == nil then return end
-
-    debug('nameserver: ' .. arg)
-    local records = dns(zone, {
-        record='AXFR',
-        nameserver=arg .. ':53',
-        tcp=true,
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        iter_axfr(zone, records[i])
-        if last_err() then return end
-    end
-end
-
-function iter_ns(zone, arg)
-    if arg == nil then return end
-
-    local records = dns(arg, {
-        record='A',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        r = records[i][2]
-        iter_a(zone, r['A'])
-        if last_err() then return end
-    end
-end
-
-function run(arg)
-    local records = dns(arg['value'], {
-        record='NS',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        local r = records[i][2]
-        iter_ns(arg['value'], r['NS'])
-        if last_err() then return end
-    end
-end

modules/dev/cname-harvest.lua

@@ -1,48 +0,0 @@
--- Description: Query for CNAMES to find subdomains
--- Version: 0.2.0
--- Source: subdomains
--- License: GPL-3.0
-
-function iter(r)
-    if r == nil then
-        return
-    end
-
-    m = regex_find("(.+)\\.$", r)
-    if last_err() then return end
-
-    if m == nil then
-        return
-    end
-    r = m[2]
-
-    domain = psl_domain_from_dns_name(r)
-    if last_err() then return end
-
-    domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=r,
-        })
-        if last_err() then return end
-    end
-end
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    if records['error'] ~= nil then return end
-    records = records['answers']
-
-    for i=1, #records do
-        r = records[i][2]
-        iter(r['CNAME'])
-        if last_err() then return end
-    end
-end

modules/dev/ctlogs.lua

@@ -1,101 +0,0 @@
--- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.4.0
--- Source: domains
--- License: GPL-3.0
-
-function each_name(name)
-    local domain_id, psl_domain
-
-    if seen[name] == 1 then
-        return
-    end
-    seen[name] = 1
-    debug(name)
-
-    if name:find('*.') == 1 then
-        -- ignore wildcard domains
-        return
-    end
-
-    -- the cert might be valid for subdomains that do not belong to the
-    -- domain we started with
-    psl_domain = psl_domain_from_dns_name(name)
-    domain_id = domains[psl_domain]
-    if domain_id == nil then
-        if any_domain then
-            -- unknown domains should be added to database
-            domain_id = db_add('domain', {
-                value=psl_domain,
-            })
-        else
-            -- only use domains that are already in scope
-            domain_id = db_select('domain', psl_domain)
-        end
-
-        -- if we didn't get a valid id, skip
-        if domain_id == nil then
-            return
-        end
-
-        domains[psl_domain] = domain_id
-    end
-
-    db_add('subdomain', {
-        domain_id=domain_id,
-        value=name,
-    })
-end
-
-function run(arg)
-    full = getopt('full') ~= nil
-    any_domain = getopt('any-domain') ~= nil
-
-    domains = {}
-    domains[arg['value']] = arg['id']
-
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://crt.sh/', {
-        query={
-            q='%.' .. arg['value'],
-            output='json'
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    certs = json_decode(resp['text'])
-    if last_err() then return end
-
-    seen = {}
-
-    for i=1, #certs do
-        c = certs[i]
-        debug(c)
-
-        if full then
-            -- fetch certificate
-            id = c['min_cert_id']
-            req = http_request(session, 'GET', 'https://crt.sh/', {
-                query={
-                    d=id .. '', -- TODO: find nicer way for tostring
-                }
-            })
-            resp = http_send(req)
-            if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-            -- iterate over all valid names
-            crt = x509_parse_pem(resp['text'])
-            if last_err() then return end
-            names = crt['valid_names']
-
-            for j=1, #names do
-                each_name(names[j])
-            end
-        else
-            each_name(c['name_value'])
-        end
-    end
-end

modules/dev/ddwrt-dhcp.lua

@@ -1,88 +0,0 @@
--- Description: Export dhcp leases from ddwrt webinterface
--- Version: 0.1.0
--- License: GPL-3.0
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    skip_redacted = not getopt('use-redacted')
-
-    router = getopt('router') -- http://192.0.2.1/
-    if not router then
-        return 'router option is missing (http://192.0.2.1/)'
-    end
-    username = getopt('user')
-    password = getopt('password')
-
-    options = {}
-    if username and password then
-        options['basic_auth'] = {username, password}
-    end
-
-    -- request status page
-    session = http_mksession()
-    url = url_join(router, '/Info.live.htm')
-    req = http_request(session, 'GET', url, options)
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then
-        return 'http error: ' .. resp['status']
-    end
-
-    txt = resp['text']
-    debug(txt)
-
-    -- extract leases from response
-    dhcp_section = regex_find('\\{dhcp_leases:: ([^\\}]+)\\}', txt)
-    if last_err() then return end
-    if not dhcp_section then
-        return 'Failed to get dhcp lease section'
-    end
-
-    leases = regex_find_all('\'([^\']+)\',\'([^\']+)\',\'([^\']+)\',\'[^\']+\',\'[^\']+\'', dhcp_section[2])
-    if last_err() then return end
-
-    now = datetime()
-
-    -- add devices to database
-    for i=1, #leases do
-        local hostname = leases[i][2]
-        local ipaddr = leases[i][3]
-        local macaddr = leases[i][4]
-
-        debug({
-            hostname=hostname,
-            ipaddr=ipaddr,
-            macaddr=macaddr,
-        })
-
-        if skip_redacted and macaddr:match('^xx:xx:') then
-            info('Skipping redacted macaddr')
-        else
-            local device = {
-                value=macaddr,
-                last_seen=now,
-            }
-            if hostname ~= '*' then
-                device['hostname'] = hostname
-            end
-
-            local device_id = db_add('device', device)
-
-            db_add_ttl('network-device', {
-                network_id=network_id,
-                device_id=device_id,
-                ipaddr=ipaddr,
-                last_seen=now,
-            }, 120)
-        end
-    end
-end

modules/dev/dns-ptr.lua

@@ -1,30 +0,0 @@
--- Description: Run reverse dns lookups
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    if arg['family'] == '4' then
-        m = regex_find('^(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)$', arg['value'])
-
-        q = m[5] .. '.' .. m[4] .. '.' .. m[3] .. '.' .. m[2] .. '.in-addr.arpa'
-        debug('Resolving: ' .. q)
-
-        records = dns(q, {
-            record='PTR',
-        })
-        if last_err() then return end
-        if records['error'] ~= nil then return end
-        records = records['answers']
-
-        for i=1, #records do
-            r = records[i][2]
-            if r['PTR'] then
-                db_update('ipaddr', arg, {
-                    reverse_dns=r['PTR'],
-                })
-                if last_err() then return end
-            end
-        end
-    end
-end

modules/dev/dns-resolve.lua

@@ -1,41 +0,0 @@
--- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.2.0
--- Source: subdomains
--- License: GPL-3.0
-
-function run(arg)
-    records = dns(arg['value'], 'A')
-    if last_err() then return end
-
-    -- update subdomain
-    resolvable = records['error'] == nil
-    if arg['resolvable'] ~= resolvable then
-        -- TODO: pass arg to function as well
-        db_update('subdomain', arg, {
-            resolvable=resolvable
-        })
-    end
-
-    if not resolvable then
-        return
-    end
-
-    records = records['answers']
-
-    for i=1, #records do
-        r = records[i][2]
-        if r['A'] ~= nil then
-            ipaddr_id = db_add('ipaddr', {
-                family='4',
-                value=r['A'],
-            })
-            if last_err() then return end
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=arg['id'],
-                ip_addr_id=ipaddr_id,
-            })
-            if last_err() then return end
-        end
-    end
-end

modules/dev/exif.lua

@@ -1,12 +0,0 @@
--- Description: Extract exif data from images
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: images
-
-function run(arg)
-    exif = img_exif(arg['value'])
-    if last_err() then return end
-    debug(exif)
-
-    db_update('image', arg, exif)
-end

modules/dev/geoip.lua

@@ -1,10 +0,0 @@
--- Description: Run a geoip lookup for an ip address
--- Version: 0.1.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    lookup = geoip_lookup(arg['value'])
-    if last_err() then return end
-    db_update('ipaddr', arg, lookup)
-end

modules/dev/git-webroot.lua

@@ -1,28 +0,0 @@
--- Description: Search for git checkouts in webroot
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    url = url_join(arg['value'], '.git/HEAD')
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    reply = http_send(req)
-    if last_err() then return end
-
-    if reply['status'] ~= 200 then
-        return
-    end
-
-    if not regex_find('^ref: ', reply['text']) then
-        return
-    end
-
-    db_add('url', {
-        subdomain_id=arg['subdomain_id'],
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-    })
-end

modules/dev/github.lua

@@ -1,107 +0,0 @@
--- Description: Collect data from github profiles
--- Version: 0.1.0
--- Source: accounts:github.com
--- License: GPL-3.0
-
-function api_get(url)
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    -- TODO: set_error(?)
-    if resp['status'] == 403 then return 'ratelimit exceeded' end
-    if resp['status'] ~= 200 then return 'invalid status code' end
-
-    local data = json_decode(resp['text'])
-    if last_err() then return end
-    return data
-end
-
-function import_gpg(url)
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-
-    local key = pgp_pubkey_armored(resp['text'])
-    if not key['uids'] then return end
-
-    for i=1, #key['uids'] do
-        local k = key['uids'][i]
-        debug(k)
-        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", k)
-        if m then
-            db_add('email', {
-                value=m[3],
-                displayname=m[2],
-            })
-        end
-    end
-end
-
-function scan4email(username)
-    local url = 'https://api.github.com/users/' .. username .. '/repos'
-    local repos = api_get(url)
-    if last_err() then return end
-
-    -- XXX: 'https://api.github.com/users/' .. username .. '/events/public?page=0&per_page=100' is faster but less accurate
-
-    for i=1, #repos do
-        local repo = repos[i]
-        debug(repo)
-        local commits = api_get(repo['url'] .. '/commits')
-        if last_err() then return end
-
-        for j=1, #commits do
-            local commit = commits[j]
-            debug(commit)
-
-            if commit['author'] and commit['author']['login'] == username then
-                local name = commit['commit']['author']['name']
-                local email = commit['commit']['author']['email']
-                db_add('email', {
-                    value=email,
-                    displayname=name,
-                })
-                return email
-            end
-
-            if commit['committer'] and commit['committer']['login'] == username then
-                local name = commit['commit']['committer']['name']
-                local email = commit['commit']['committer']['email']
-                db_add('email', {
-                    value=email,
-                    displayname=name,
-                })
-                return email
-            end
-        end
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    local url = 'https://api.github.com/users/' .. arg['username']
-
-    local data = api_get(url)
-    if last_err() then return end
-    debug(data)
-
-    -- company = data['company']
-    -- location = data['location']
-    -- homepage = data['blog']
-
-    url = 'https://github.com/' .. arg['username'] .. '.gpg'
-    import_gpg(url)
-    if last_err() then return end
-
-    local email = data['email']
-    if not email and not arg['email'] then
-        email = scan4email(arg['username'])
-        if last_err() then return end
-    end
-
-    db_update('account', arg, {
-        url=data['html_url'],
-        displayname=data['name'],
-        email=email,
-    })
-end

modules/dev/hackertarget-subdomains.lua

@@ -1,27 +0,0 @@
--- Description: Query hackertarget for subdomains of a domain
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.hackertarget.com/hostsearch/', {
-        query={
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    m = regex_find_all("([^,]+),.+\\n?", resp['text'])
-
-    for i=1, #m do
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=m[i][2]
-        })
-    end
-end

modules/dev/home-assistant-devices.lua

@@ -1,56 +0,0 @@
--- Description: Query device location from home assistant
--- Version: 0.1.0
--- License: GPL-3.0
--- Keyring-Access: home-assistant
-
-function run()
-    -- parsing options
-    instance = getopt('instance')
-    if not instance then
-        return 'instance option is missing'
-    end
-
-    host = url_parse(instance)
-    if last_err() then return end
-    host = host['host']
-
-    entity = getopt('entity')
-    if not entity then
-        return 'entity option is missing'
-    end
-
-    -- fetching credentials
-    creds = keyring('home-assistant:' .. host)
-    if creds[1] == nil then
-        profile = url_join(instance, 'profile')
-        return 'missing home-assistant:' .. host .. ' Long-Lived Access Token, open ' .. profile
-    end
-    token = creds[1]['secret_key']
-
-    headers = {}
-    headers['Authorization'] = 'Bearer ' .. token
-    headers['Content-Type'] = 'application/json'
-
-    -- requesting status
-    session = http_mksession()
-    url = url_join(instance, 'api/states/' .. entity)
-    req = http_request(session, 'GET', url, {
-        headers=headers
-    })
-    r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then
-        return 'http error: ' .. r['status']
-    end
-
-    m = json_decode(r['text'])
-    if last_err() then return end
-    debug(m)
-
-    info({
-        gps_accuracy=m['attributes']['gps_accuracy'],
-        longitude=m['attributes']['longitude'],
-        latitude=m['attributes']['latitude'],
-        last_updated=m['last_updated'],
-    })
-end

modules/dev/images.lua

@@ -1,9 +0,0 @@
--- Description: Parse image metadata
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: images
-
-function run(arg)
-    local img = img_load(arg['value'])
-    db_update('image', arg, img)
-end

modules/dev/instagram.lua

@@ -1,150 +0,0 @@
--- Description: Collect data from instagram profiles
--- Version: 0.1.0
--- Source: accounts:instagram.com
--- License: GPL-3.0
-
-PAGE_SIZE = 50
-
-function get_shared_data(html)
-    local s = html_select_list(html, 'script')
-
-    for i=1, #s do
-        local m = regex_find('^window\\._sharedData = (.+);$', s[i]['text'])
-        if m then
-            return json_decode(m[2])
-        end
-    end
-end
-
-function sign_request(rhx, json_params)
-    local magic = rhx .. ':' .. json_params
-    local x_instagram_gis = hex(md5(magic))
-    return x_instagram_gis
-end
-
-function download_image(node)
-    local url = node['display_url']
-    debug(url)
-
-    local req = http_request(session, 'GET', url, {
-        into_blob=true,
-    })
-    local r = http_send(req)
-    if last_err() then return end
-    if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-
-    db_add('image', {
-        value=r['blob'],
-    })
-end
-
-function pull_graphql(page)
-    local end_cursor = page['page_info']['end_cursor']
-
-    for i=1, #page['edges'] do
-        -- shortcode = page['edges'][i]['shortcode']
-        local node = page['edges'][i]['node']
-        node['thumbnail_resources'] = nil
-        node['media_preview'] = nil
-        -- debug(node)
-
-        -- if node['__typename'] == 'GraphImage'
-
-        -- node['dimensions']['height']
-        -- node['dimensions']['width']
-        -- ^ not sure how to get that picture
-
-        -- node['taken_at_timestamp']
-        -- location = node['location']
-
-        local err = download_image(node)
-        if last_err() then return end
-        if err ~= nil then return err end
-
-        todo_posts = todo_posts -1
-        debug('posts left: ' .. todo_posts .. '/' .. total_posts)
-    end
-
-    if page['page_info']['has_next_page'] then
-        debug('requesting next page=' .. end_cursor)
-
-        variables = json_encode({
-            id=user['id'],
-            first=PAGE_SIZE,
-            after=end_cursor
-        })
-
-        local headers = {}
-        headers['X-Instagram-GIS'] = sign_request(rhx_gis, variables)
-
-        local req = http_request(session, 'GET', 'https://www.instagram.com/graphql/query/', {
-            query={
-                query_hash='42323d64886122307be10013ad2dcc44',
-                variables=variables,
-            },
-            headers=headers,
-        })
-        r = http_send(req)
-        if last_err() then return end
-        if r['status'] ~= 200 then return 'http error: ' .. r['status'] end
-
-        x = json_decode(r['text'])
-        if last_err() then return end
-        return pull_graphql(x['data']['user']['edge_owner_to_timeline_media'])
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    local url = 'https://www.instagram.com/' .. arg['username'] .. '/'
-    local req = http_request(session, 'GET', url, {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'invalid status code' end
-    local html = resp['text']
-
-    local ld = html_select(html, 'script[type="application/ld+json"]')
-    if last_err() then return end
-
-    local ld = json_decode(ld['text'])
-    if last_err() then return end
-    --debug(ld)
-
-    if ld['email'] then
-        db_add('email', {
-            value=ld['email'],
-        })
-    end
-
-    -- homepage=ld['url']
-
-    db_update('account', arg, {
-        displayname=ld['name'],
-        email=ld['email'],
-        url=url,
-    })
-
-    -- download images
-    local sd = get_shared_data(html)
-    if last_err() then return end
-    -- debug(sd)
-
-    rhx_gis = sd['rhx_gis']
-    user = sd['entry_data']['ProfilePage'][1]['graphql']['user']
-
-    -- user['full_name']
-    -- user['id']
-    -- user['is_business_account']
-    -- user['is_private']
-    -- user['is_verified']
-    -- user['has_blocked_viewer']
-    -- user['connected_fb_page']
-    -- user['country_block']
-
-    local page = user['edge_owner_to_timeline_media']
-    total_posts = page['count']
-    todo_posts = total_posts
-
-    -- TODO: fast-update abort if image has been downloaded already
-    return pull_graphql(page)
-end

modules/dev/isc-dhcpd-leases.lua

@@ -1,78 +0,0 @@
--- Description: Parse isc-dhcpd dhcpd.leases(5)
--- Version: 0.2.0
--- License: GPL-3.0
-
--- cat /var/lib/dhcpd/dhcpd.leases
-
-function add(lease)
-    if not lease['active'] then return end
-
-    now = datetime()
-
-    device_id = db_add('device', {
-        value=lease['mac'],
-        hostname=lease['hostname'],
-        last_seen=now,
-    })
-    if last_err() then return end
-
-    db_add_ttl('network-device', {
-        network_id=network_id,
-        device_id=device_id,
-        ipaddr=lease['ipaddr'],
-        last_seen=now,
-    }, 180)
-    if last_err() then return end
-end
-
-function each_line(x)
-    debug(x)
-    m = regex_find('^lease (\\S+) \\{\n$', x)
-    if m then
-        lease = {}
-        debug('ipaddr=' .. m[2])
-        lease['ipaddr'] = m[2]
-    end
-    m = regex_find('^\\s*hardware ethernet (\\S+);\n$', x)
-    if m then
-        debug('mac=' .. m[2])
-        lease['mac'] = m[2]
-    end
-    m = regex_find('^\\s*client-hostname \"(.+)\";\n$', x)
-    if m then
-        debug('hostname=' .. m[2])
-        lease['hostname'] = m[2]
-    end
-    m = regex_find('^\\s*binding state active;\n$', x)
-    if m then
-        debug('active=true')
-        lease['active'] = true
-    end
-    m = regex_find('^\\}\n$', x)
-    if m then
-        add(lease)
-    end
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        if not regex_find('^\\s*(#.*|\\s*)\n$', x) then
-            each_line(x)
-        end
-    end
-end

modules/dev/iw-station-dump.lua

@@ -1,77 +0,0 @@
--- Description: Parse iw station dump
--- Version: 0.2.0
--- License: GPL-3.0
-
--- iw dev wlan0 station dump
-
-function add(client)
-    if
-        client['authenticated'] == 'yes' and
-        client['authorized'] == 'yes' and
-        client['mac']
-    then
-        debug(client)
-
-        now = datetime()
-
-        device_id = db_add('device', {
-            value=client['mac'],
-            last_seen=now,
-        })
-        if last_err() then return end
-
-        db_add_ttl('network-device', {
-            network_id=network_id,
-            device_id=device_id,
-            last_seen=now,
-        }, 180)
-        if last_err() then return end
-    end
-
-    client = nil
-end
-
-function each_line(x)
-    debug(x)
-    m = regex_find('^Station (\\S+)', x)
-    if m then
-        if client then
-            add(client)
-        end
-        client = {}
-        client['mac'] = m[2]
-        debug('mac=' .. m[2])
-    end
-
-    m = regex_find('^\\s+([^:]+):\\s*(.+)\n$', x)
-    if m and client then
-        client[m[2]] = m[3]
-        debug(m[2] .. '=' .. m[3])
-    end
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    client = nil
-    while true do
-        x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        each_line(x)
-    end
-
-    if client then
-        add(client)
-    end
-end

modules/dev/keybase-domains.lua

@@ -1,28 +0,0 @@
--- Description: Find keybase proofs for domains
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: domains
-
-function run(arg)
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
-        query={
-            domain=arg['value'],
-        }
-    })
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    x = json_decode(resp['text'])
-    if last_err() then return end
-    debug(x)
-
-    if x['them'][1] == nil then return end
-    them = x['them'][1]
-
-    db_add('account', {
-        service='keybase.io',
-        username=them['basics']['username'],
-    })
-end

modules/dev/keybase-profiles.lua

@@ -1,44 +0,0 @@
--- Description: Find keybase proofs for online accounts
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: accounts
-
-function run(arg)
-    service = arg['service']
-    if service == 'twitter.com' then
-        service = 'twitter'
-    elseif service == 'github.com' then
-        service = 'github'
-    elseif service == 'reddit.com' then
-        service = 'reddit'
-    elseif service == 'news.ycombinator.com' then
-        service = 'hackernews'
-    elseif service == 'facebook.com' then
-        service = 'facebook'
-    else
-        return
-    end
-
-    query = {}
-    query[service] = arg['username']
-
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
-        query=query,
-    })
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    x = json_decode(resp['text'])
-    if last_err() then return end
-    debug(x)
-
-    if x['them'][1] == nil then return end
-    them = x['them'][1]
-
-    db_add('account', {
-        service='keybase.io',
-        username=them['basics']['username'],
-    })
-end

modules/dev/keybase.lua

@@ -1,85 +0,0 @@
--- Description: Collect accounts and emails from keybase accounts
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: accounts:keybase.io
-
-function extract_mails(pubkey)
-    for j=1, #pubkey['uids'] do
-        local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
-        if m then
-            db_add('email', {
-                value=m[3],
-                displayname=m[2],
-            })
-        end
-    end
-end
-
-function add_domain(dns)
-    local domain = psl_domain_from_dns_name(dns)
-    if last_err() then return end
-
-    local domain_id = db_add('domain', {
-        value=domain,
-    })
-    if last_err() then return end
-    if domain_id == nil then return end
-
-    if domain ~= dns then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=dns,
-        })
-    end
-end
-
-function run(arg)
-    session = http_mksession()
-    req = http_request(session, 'GET', 'https://keybase.io/_/api/1.0/user/lookup.json', {
-        query={
-            usernames=arg['username'],
-        }
-    })
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    x = json_decode(resp['text'])
-    if last_err() then return end
-    debug(x)
-
-    if x['them'][1] == nil then return end
-    them = x['them'][1]
-
-    -- update keybase profile
-    db_update('account', arg, {
-        displayname=them['profile']['full_name'],
-        url='https://keybase.io/'..arg['username'],
-    })
-
-    -- collect emails
-    pubkey = pgp_pubkey_armored(them['public_keys']['primary']['bundle'])
-    debug(pubkey)
-    extract_mails(pubkey)
-
-    -- collect profiles
-    profiles = them['proofs_summary']['all']
-
-    for i=1, #profiles do
-        profile = profiles[i]
-        debug(profile)
-
-        if
-            profile['proof_type'] == 'generic_web_site' or
-            profile['proof_type'] == 'dns'
-        then
-            add_domain(profile['nametag'])
-        else
-            db_add('account', {
-                service=profile['proof_type'],
-                username=profile['nametag'],
-                url=profile['service_url'],
-            })
-        end
-    end
-end

modules/dev/namechk.lua

@@ -1,85 +0,0 @@
--- Description: Find accounts by username with namechk.com
--- Version: 0.1.0
--- Source: accounts
--- License: GPL-3.0
-
-function get_services(html)
-    local divs = html_select_list(html, '.service')
-    if last_err() then return end
-
-    local services = {}
-
-    for i=1, #divs do
-        services[i] = divs[i]['attrs']['data-name']
-    end
-
-    return services
-end
-
-function run(arg)
-    -- setup session
-    local session = http_mksession()
-    local req = http_request(session, 'GET', 'https://namechk.com/', {})
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    local token = html_select(resp['text'], 'input[name="authenticity_token"]')
-    local auth_token = token['attrs']['value']
-
-        local headers = {}
-        headers['X-CSRF-Token'] = authenticity_token
-
-    local services = get_services(resp['text'])
-    debug({
-        auth_token=auth_token,
-        services=services,
-    })
-
-    -- trigger the scan
-    local req = http_request(session, 'POST', 'https://namechk.com/', {
-            headers=headers,
-        form={
-            authenticity_token=auth_token,
-            q=arg['username'],
-        }
-    })
-    local resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-    debug(resp)
-
-    local scan = json_decode(resp['text'])
-    if last_err() then return end
-    local scan_token = scan['valid']
-
-    -- get results
-    for i=1, #services do
-        debug(services[i])
-
-        local req = http_request(session, 'POST', 'https://namechk.com/services/check', {
-            headers=headers,
-            form={
-                token=scan_token,
-                fat=auth_token,
-                service=services[i],
-            }
-        })
-        local resp = http_send(req)
-        if last_err() then return end
-
-        if resp['status'] == 200 then
-            local acc = json_decode(resp['text'])
-            if last_err() then return end
-            debug(acc)
-
-            if acc ~= nil and not acc['available'] and acc['status'] == 'unavailable' then
-                db_add('account', {
-                    service=services[i],
-                    username=arg['username'],
-                    url=acc['callback_url'],
-                })
-            end
-        end
-    end
-end

modules/dev/nudity.lua

@@ -1,12 +0,0 @@
--- Description: Scan collected images for nudity
--- Version: 0.1.0
--- License: GPL-3.0
--- Source: images
-
-function run(arg)
-    local nudity = img_nudity(arg['value'])
-    debug(nudity)
-    db_update('image', arg, {
-        nudity=nudity['score'],
-    })
-end

modules/dev/otx-subdomains.lua

@@ -1,29 +0,0 @@
--- Description: Query alienvault otx passive dns for subdomains of a domain
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    url = 'https://otx.alienvault.com/api/v1/indicators/domain/' .. arg['value'] .. '/passive_dns'
-
-    req = http_request(session, 'GET', url, {})
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['passive_dns']
-
-    for i=1, #o do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x['hostname'],
-        })
-    end
-end

modules/dev/passive-arp.lua

@@ -1,79 +0,0 @@
--- Description: Passive arp-scanner with sniffglue
--- Version: 0.1.0
--- License: GPL-3.0
-
--- sudo sniffglue -jv enp0s25
-
-function each_frame(frame)
-    if not frame['Ether'] then return end
-
-    local arp = frame['Ether'][2]['Arp']
-    if not arp then return end
-
-    if arp['Request'] then
-        arp = arp['Request']
-    elseif arp['Reply'] then
-        arp = arp['Reply']
-    else
-        -- unknown, abort
-        return
-    end
-
-    debug(arp)
-
-    -- TODO: this might change to a string in the future
-    local mac = mac(arp['src_mac'])
-    local ipaddr = arp['src_addr']
-    debug({src_mac=mac, src_addr=ipaddr})
-
-    local now = datetime()
-
-    local device_id = db_add('device', {
-        value=mac,
-        last_seen=now,
-    })
-    if last_err() then return end
-
-    db_add_ttl('network-device', {
-        network_id=network_id,
-        device_id=device_id,
-        ipaddr=ipaddr,
-        last_seen=now,
-    }, 120)
-    if last_err() then return end
-end
-
-function mac(m)
-    return
-        hex({m[1]}) .. ':' ..
-        hex({m[2]}) .. ':' ..
-        hex({m[3]}) .. ':' ..
-        hex({m[4]}) .. ':' ..
-        hex({m[5]}) .. ':' ..
-        hex({m[6]})
-end
-
-function run()
-    network = getopt('network')
-    if not network then
-        return 'network option is missing'
-    end
-
-    network_id = db_select('network', network)
-    if not network_id then
-        return 'network not found in database'
-    end
-
-    while true do
-        local x = stdin_readline()
-        if x == nil then
-            break
-        end
-
-        local frame = json_decode(x)
-        if last_err() then return end
-
-        each_frame(frame)
-        if last_err() then return end
-    end
-end

modules/dev/passive-spider.lua

@@ -1,60 +0,0 @@
--- Description: Scrape known http responses for urls
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function entry(parent, href)
-    -- TODO: parse mailto:foo@example.com?subject=asdf
-    -- TODO: parse tel:+4912345
-    -- TODO: allow discovering 3rd-party domains
-    -- TODO: maybe record urls as well
-
-    local psl, parts, url, host
-
-    if href == nil then
-        return
-    end
-
-    url = url_join(parent, href)
-    if last_err() then return clear_err() end
-    if url:match('^https?://') == nil then
-        return
-    end
-
-    parts = url_parse(url)
-    if last_err() then return end
-    host = parts['host']
-    psl = psl_domain_from_dns_name(host)
-
-
-    domain_id = db_select('domain', psl)
-    if domain_id ~= nil then
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=host,
-        })
-    end
-end
-
-function run(arg)
-    if arg['body'] == nil or #arg['body'] == 0 then
-        return
-    end
-
-    body = utf8_decode(arg['body'])
-    if last_err() then return end
-
-    links = html_select_list(body, 'a')
-    if last_err() then return end
-
-    if #links == 0 then
-        return
-    end
-
-    -- process html links
-    for i=1, #links do
-        href = links[i]['attrs']['href']
-
-        entry(arg['value'], href)
-    end
-end

modules/dev/pgp-keyserver.lua

@@ -1,54 +0,0 @@
--- Description: Query pgp keyserver for email addresses
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    --lookup_url = 'https://pgp.mit.edu/pks/lookup'
-    lookup_url = 'https://sks-keyservers.net/pks/lookup'
-
-    req = http_request(session, 'GET', lookup_url, {
-        query={
-            search=arg['value'],
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    links = html_select_list(resp['text'], 'a')
-    for i=1, #links do
-        href = links[i]['attrs']['href']
-
-        if href:find('/pks/lookup%?op=get&search=') == 1 then
-            url = url_join(lookup_url, href)
-
-            req = http_request(session, 'GET', url, {})
-
-            resp = http_send(req)
-            -- TODO: do not abort script if one attempt fails
-            if last_err() then return end
-            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-            pubkey = pgp_pubkey_armored(resp['text'])
-
-            -- print(pubkey)
-
-            -- TODO: ensure at least one email matches our target domain
-            if pubkey['uids'] then
-                for j=1, #pubkey['uids'] do
-                    local m = regex_find("(.+) <([^< ]+@[^< ]+)>$", pubkey['uids'][j])
-                    if m then
-                        db_add('email', {
-                            value=m[3],
-                            displayname=m[2],
-                        })
-                    end
-                end
-            end
-        end
-    end
-end

modules/dev/phpmyadmin.lua

@@ -1,103 +0,0 @@
--- Description: Search for phpmyadmin
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    paths = {
-        "phpmyadmin/index.php",
-        "phpMyAdmin/index.php",
-        "pmd/index.php",
-        "pma/index.php",
-        "PMA/index.php",
-        "PMA2/index.php",
-        "pmamy/index.php",
-        "pmamy2/index.php",
-        "mysql/index.php",
-        "admin/index.php",
-        "db/index.php",
-        "dbadmin/index.php",
-        "web/phpMyAdmin/index.php",
-        "admin/pma/index.php",
-        "admin/PMA/index.php",
-        "admin/mysql/index.php",
-        "admin/mysql2/index.php",
-        "admin/phpmyadmin/index.php",
-        "admin/phpMyAdmin/index.php",
-        "admin/phpmyadmin2/index.php",
-        "mysqladmin/index.php",
-        "mysql-admin/index.php",
-        "mysql_admin/index.php",
-        "phpadmin/index.php",
-        "phpAdmin/index.php",
-        "phpmyadmin0/index.php",
-        "phpmyadmin1/index.php",
-        "phpmyadmin2/index.php",
-        "phpMyAdmin-4.4.0/index.php",
-        "myadmin/index.php",
-        "myadmin2/index.php",
-        "xampp/phpmyadmin/index.php",
-        "phpMyadmin_bak/index.php",
-        "www/phpMyAdmin/index.php",
-        "tools/phpMyAdmin/index.php",
-        "phpmyadmin-old/index.php",
-        "phpMyAdminold/index.php",
-        "phpMyAdmin.old/index.php",
-        "pma-old/index.php",
-        "claroline/phpMyAdmin/index.php",
-        "typo3/phpmyadmin/index.php",
-        "phpma/index.php",
-        "phpmyadmin/phpmyadmin/index.php",
-        "phpMyAdmin/phpMyAdmin/index.php",
-        "phpMyAbmin/index.php",
-        "phpMyAdmin__/index.php",
-        "phpMyAdmin+++---/index.php",
-        "v/index.php",
-        "phpmyadm1n/index.php",
-        "phpMyAdm1n/index.php",
-        "shaAdmin/index.php",
-        "phpMyadmi/index.php",
-        "phpMyAdmion/index.php",
-        "MyAdmin/index.php",
-        "phpMyAdmin1/index.php",
-        "phpMyAdmin123/index.php",
-        "pwd/index.php",
-        "phpMyAdmina/index.php",
-        "program/index.php",
-        "shopdb/index.php",
-        "phppma/index.php",
-        "phpmy/index.php",
-        "mysql/admin/index.php",
-        "mysql/dbadmin/index.php",
-        "mysql/sqlmanager/index.php",
-        "mysql/mysqlmanager/index.php",
-        "wp-content/plugins/portable-phpmyadmin/wp-pma-mod/index.php",
-    }
-
-    session = http_mksession()
-
-    for i=1, #paths do
-        p = paths[i]
-        url = url_join(arg['value'], p)
-        debug(url)
-
-        req = http_request(session, 'GET', url, {
-            timeout=5000
-        })
-        reply = http_send(req)
-        debug(reply)
-
-        if last_err() then
-            clear_err()
-        else
-            if reply['status'] == 200 then
-                db_add('url', {
-                    subdomain_id=arg['subdomain_id'],
-                    value=url,
-                    status=reply['status'],
-                    body=reply['text'],
-                })
-            end
-        end
-    end
-end

modules/dev/smtp-check.lua

@@ -1,88 +0,0 @@
--- Description: Verify email address by asking the smtp server
--- Version: 0.1.0
--- Source: emails
--- License: GPL-3.0
-
-function find_mx(domain)
-    local records, i, r
-
-    records = dns(domain, {
-        record='MX',
-    })
-    if last_err() then return end
-    if records['error'] ~= nil then return end
-    records = records['answers']
-    -- debug(records)
-
-    for i=1, #records do
-        r = records[i][2]['MX']
-        if r then
-            debug('mx: ' .. r[2])
-            return r[2]
-        end
-    end
-end
-
-function run(arg)
-    -- extract domain
-    domain = arg['value']:match('@(.*)')
-    if doman ~= nil then
-        -- malformed domain
-        return
-    end
-
-    -- mx lookup
-    mx = find_mx(domain)
-    if last_err() then return end
-    if not mx then return end
-
-    -- create connection
-    c = sock_connect(mx, 25, {})
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- send hello
-    sock_sendline(c, 'ehlo localhost')
-    if last_err() then return end
-
-    l = sock_recvline_regex(c, '^250 ')
-    if last_err() then return end
-    debug(l)
-
-    -- send email
-    sock_sendline(c, 'mail from:<root@localhost>')
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- send rcpt
-    sock_sendline(c, 'rcpt to:<' .. arg['value'] .. '>')
-    if last_err() then return end
-
-    l = sock_recvline(c)
-    if last_err() then return end
-    debug(l)
-
-    -- check status
-    verified = nil
-    if l:match('^2') then
-        debug('email is valid')
-        verified = true
-    elseif l:match('^5') then
-        debug('email is invalid')
-        verified = false
-    elseif l:match('^4') then
-        debug('unknown status, temporary delivery failure')
-    end
-
-    if verified ~= nil then
-        db_update('email', arg, {
-            valid=verified,
-        })
-    end
-end

modules/dev/threatminer-ipaddr.lua

@@ -1,49 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of an ip address
--- Version: 0.2.0
--- Source: ipaddrs
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- TODO: add option to filter old entries based on last_seen
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/host.php', {
-        query={
-            rt='2',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    for i=1, #o do
-        x = o[i]
-
-        domain = psl_domain_from_dns_name(x['domain'])
-        -- TODO: if this fails, skip this entry instead
-        if last_err() then return end
-
-        domain_id = db_add('domain', {
-            value=domain,
-        })
-
-        if domain_id ~= nil then
-            subdomain_id = db_add('subdomain', {
-                domain_id=domain_id,
-                value=x['domain'],
-            })
-
-            db_add('subdomain-ipaddr', {
-                subdomain_id=subdomain_id,
-                ip_addr_id=arg['id'],
-            })
-        end
-    end
-end

modules/dev/threatminer-subdomains.lua

@@ -1,32 +0,0 @@
--- Description: Query ThreatMiner passive dns for subdomains of a domain
--- Version: 0.2.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    req = http_request(session, 'GET', 'https://api.threatminer.org/v2/domain.php', {
-        query={
-            rt='5',
-            q=arg['value']
-        }
-    })
-
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-    o = o['results']
-
-    for i=1, #o do
-        x = o[i]
-
-        db_add('subdomain', {
-            domain_id=arg['id'],
-            value=x,
-        })
-    end
-end

modules/dev/thunderbird-autoconfig.lua

@@ -1,45 +0,0 @@
--- Description: Query thunderbird autoconfig db for subdomains
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-    session = http_mksession()
-
-    -- check if an autoconfig exists without disclosing our target yet
-    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/', {})
-    resp = http_send(req)
-    if last_err() then return end
-
-    if resp['status'] ~= 200 then
-        return 'index request failed'
-    end
-
-    if resp['text']:find(arg['value'], 1, true) == nil then
-        debug('no autoconfig available')
-        return
-    end
-
-    -- request config
-    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/' .. arg['value'], {})
-    resp = http_send(req)
-    if last_err() then return end
-
-    m = regex_find_all('<hostname>([^<]+)</hostname>', resp['text'])
-
-    for i=1, #m do
-        subdomain = m[i][2]
-
-        domain = psl_domain_from_dns_name(subdomain)
-        if last_err() then return end
-
-        domain_id = db_select('domain', domain)
-        if last_err() then return end
-
-        db_add('subdomain', {
-            domain_id=domain_id,
-            value=subdomain,
-        })
-        if last_err() then return end
-    end
-end

modules/dev/twilio-lookup.lua

@@ -1,48 +0,0 @@
--- Description: Retrieve additional information about a phone number
--- Version: 0.1.0
--- Source: phonenumbers
--- Keyring-Access: twilio
--- License: GPL-3.0
-
-function run(arg)
-    number = url_escape(arg['value'])
-    --url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number
-    url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number .. '?Type=carrier&Type=caller-name'
-
-    --debug(url)
-
-    key = keyring('twilio')[1]
-    if not key then
-        return 'Missing required twilio access key'
-    end
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {
-        basic_auth={key['access_key'], key['secret_key']},
-    })
-    reply = http_send(req)
-    if last_err() then return end
-
-    if reply['status'] ~= 200 then
-        return 'api returned error'
-    end
-
-    v = json_decode(reply['text'])
-    if last_err() then return end
-    debug(v)
-
-    update = {}
-    update['country'] = v['country_code']
-
-    if v['carrier'] then
-        update['carrier'] = v['carrier']['name']
-        update['line'] = v['carrier']['type']
-    end
-
-    if v['caller_name'] then
-        update['caller_name'] = v['caller_name']['caller_name']
-        update['caller_type'] = v['caller_name']['caller_type']
-    end
-
-    db_update('phonenumber', arg, update)
-end

modules/dev/url-scan.lua

@@ -1,45 +0,0 @@
--- Description: Scan subdomains for websites
--- Version: 0.3.0
--- Source: subdomains
--- License: GPL-3.0
-
-function request(subdomain_id, url)
-    req = http_request(session, 'GET', url, {
-        timeout=5000
-    })
-    reply = http_send(req)
-
-    if last_err() then
-        clear_err()
-        return
-    end
-
-    obj = {
-        subdomain_id=subdomain_id,
-        value=url,
-        status=reply['status'],
-        body=reply['text'],
-    }
-
-    redirect = reply['headers']['location']
-    if redirect then
-        obj['redirect'] = url_join(url, redirect)
-    end
-
-    db_add('url', obj)
-
-    -- debug(reply['status'])
-    -- debug(reply['headers']['location'])
-    -- debug(reply['text'])
-end
-
-function run(arg)
-    domain = arg['value']
-
-    session = http_mksession()
-
-    request(arg['id'], 'http://' .. domain .. '/')
-    if last_err() then return end
-    request(arg['id'], 'https://' .. domain .. '/')
-    if last_err() then return end
-end

modules/dev/waybackurls.lua

@@ -1,54 +0,0 @@
--- Description: Discover subdomains from wayback machine
--- Version: 0.3.0
--- Source: domains
--- License: GPL-3.0
-
-function run(arg)
-
-    domain = arg['value']
-    url = 'https://web.archive.org/cdx/search/cdx?url=*.' .. domain .. '/*&output=json&collapse=urlkey'
-
-    session = http_mksession()
-    req = http_request(session, 'GET', url, {})
-    resp = http_send(req)
-    if last_err() then return end
-    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
-
-    o = json_decode(resp['text'])
-    if last_err() then return end
-
-    -- no known urls
-    if o[1] == nil then
-        return
-    end
-
-    -- ensure the api response is still what we expect
-    if o[1][3] == nil then
-        return 'api returned unexpected json format'
-    end
-
-    seen = {}
-
-    for i=2, #o do
-        url = o[i][3]
-        debug(url)
-        parts = url_parse(url)
-
-        if last_err() then
-            clear_err()
-            error("Failed to parse url: " .. json_encode(url))
-        else
-            subdomain = parts['host']
-            subdomain, _ = subdomain:gsub('%.$', '')
-
-            if seen[subdomain] == nil then
-                db_add('subdomain', {
-                    domain_id=arg['id'],
-                    value=parts['host'],
-                })
-                if last_err() then return end
-                seen[subdomain] = 1
-            end
-        end
-    end
-end

modules/dev/well-known-uris.lua

@@ -1,61 +0,0 @@
--- Description: Scan for known /.well-known/ locations
--- Version: 0.1.0
--- Source: urls
--- License: GPL-3.0
-
-function run(arg)
-    -- https://www.iana.org/assignments/well-known-uris/well-known-uris.xhtml
-    -- https://en.wikipedia.org/wiki/List_of_/.well-known/_services_offered_by_webservers
-
-    -- TODO: check if every location causes a 200/redirect
-
-    locations = {
-        {path='security.txt'}, -- expect 200
-        {path='dnt-policy.txt'}, -- expect 200
-        {path='caldav', redirect=true}, -- expect redirect
-        {path='autoconfig/mail/config-v1.1.xml'}, -- expect 200
-        {path='assetlinks.json'}, -- expect 200
-        {path='apple-app-site-association'},  -- expect 200
-        {path='keybase.txt'}, -- expect 200
-        {path='apple-developer-merchantid-domain-association'}, -- expect 200
-        {path='openpgpkey'}, -- expect 200
-        {path='change-password', redirect=true}, -- expect redirect
-    }
-
-    session = http_mksession()
-
-    for i=1, #locations do
-        path = locations[i]['path']
-        expect_redirect = locations[i]['redirect']
-
-        url = url_join(arg['value'], '/.well-known/' .. path)
-        debug(url)
-
-        req = http_request(session, 'GET', url, {
-            timeout=5000,
-        })
-        reply = http_send(req)
-        debug(reply)
-
-        if last_err() then
-            clear_err()
-        else
-            status = reply['status']
-            if (status == 200 and not expect_redirect) or (expect_redirect and status >= 300 and status < 400) then
-                obj = {
-                    subdomain_id=arg['subdomain_id'],
-                    value=url,
-                    status=reply['status'],
-                    body=reply['text'],
-                }
-
-                redirect = reply['headers']['location']
-                if redirect then
-                    obj['redirect'] = url_join(url, redirect)
-                end
-
-                db_add('url', obj)
-            end
-        end
-    end
-end

modules/harness/activity-ping-once.lua

@@ -0,0 +1,26 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    db_activity({
+        topic=topic,
+        time=sn0int_time(),
+        uniq=uniq,
+        latitude=lat,
+        longitude=lon,
+        radius=radius,
+        content={
+            msg='ohai',
+        },
+    })
+end

modules/harness/activity-ping.lua

@@ -0,0 +1,29 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    while true do
+        db_activity({
+            topic=topic,
+            time=sn0int_time(),
+            uniq=uniq,
+            latitude=lat,
+            longitude=lon,
+            radius=radius,
+            content={
+                msg='ohai',
+            },
+        })
+        sleep(5)
+    end
+end

modules/harness/add-all.lua

@@ -0,0 +1,107 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('adding domain')
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+    if last_err() then return end
+
+    info('adding subdomain')
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value='example.com',
+    })
+    if last_err() then return end
+
+    info('adding ipaddr')
+    ipaddr_id = db_add('ipaddr', {
+        value='192.0.2.1',
+    })
+    if last_err() then return end
+
+    info('adding device')
+    device_id = db_add('device', {
+        value='ff:ff:ff:ff:ff:ff',
+    })
+    if last_err() then return end
+
+    info('adding network')
+    network_id = db_add('network', {
+        value='myssid',
+    })
+    if last_err() then return end
+
+    info('adding email')
+    email_id = db_add('email', {
+        value='foo@example.com',
+    })
+    if last_err() then return end
+
+    info('adding phonenumber')
+    phonenumber_id = db_add('phonenumber', {
+        value='+4912345678',
+    })
+    if last_err() then return end
+
+    info('adding breach')
+    breach_id = db_add('breach', {
+        value='hack the planet',
+    })
+    if last_err() then return end
+
+    info('adding account')
+    account_id = db_add('account', {
+        service='github.com',
+        username='kpcyrd',
+    })
+    if last_err() then return end
+
+    info('adding image')
+    blob = create_blob('abc')
+    image_id = db_add('image', {
+        value=blob,
+    })
+    if last_err() then return end
+
+    info('adding port')
+    port_id = db_add('port', {
+        ip_addr_id=ipaddr_id,
+        ip_addr='192.0.2.1',
+        port=443,
+        protocol='tcp',
+        status='open',
+    })
+    if last_err() then return end
+
+    info('adding url')
+    url_id = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://www.example.com/a/b',
+        body='<html></html>',
+    })
+    if last_err() then return end
+
+    info('adding breach_email')
+    db_add('breach-email', {
+        breach_id=breach_id,
+        email_id=email_id,
+    })
+    if last_err() then return end
+
+    info('adding network_device')
+    db_add('network-device', {
+        network_id=network_id,
+        device_id=device_id,
+    })
+    if last_err() then return end
+
+    info('adding subdomain_ipaddr')
+    db_add('subdomain-ipaddr', {
+        subdomain_id=subdomain_id,
+        ip_addr_id=ipaddr_id,
+    })
+    if last_err() then return end
+end

modules/harness/commit-log.lua

@@ -0,0 +1,26 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- git log -s --format='%H %ci'
+
+function run()
+    while true do
+        local x = stdin_readline()
+        if x == nil then
+            break
+        end
+        local m = regex_find('^(\\S+) (.+)', x)
+        if m then
+            time = strptime('%Y-%m-%d %T %z', m[3])
+            time = sn0int_time_from(time)
+
+            db_activity({
+                topic='harness/sn0int-commit:dummy',
+                time=time,
+                uniq=m[2],
+                content={},
+            })
+        end
+    end
+end

modules/harness/dummy-resolve.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: subdomains
+
+function run(arg)
+    db_update('subdomain', arg, {
+        resolvable=true,
+    })
+end

modules/harness/dummy-subdomains.lua

@@ -0,0 +1,13 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    for i=1, 20 do
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=http_mksession() .. '.' .. arg['value'],
+        })
+    end
+end

modules/harness/geo-polygon-contains.lua

@@ -0,0 +1,74 @@
+-- Description: demonstrate geofencing with polygons
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    hamburg = {
+        { lat=53.63975308945899,  lon=9.764785766601562  },
+        { lat=53.59494998253459,  lon=9.827270507812     },
+        { lat=53.663153974456456, lon=9.9151611328125    },
+        { lat=53.65582987649682,  lon=9.976272583007812  },
+        { lat=53.68613523817129,  lon=9.992752075195312  },
+        { lat=53.68674518938816,  lon=10.051460266113281 },
+        { lat=53.72495117617815,  lon=10.075492858886719 },
+        { lat=53.71946627930625,  lon=10.118408203125    },
+        { lat=53.743635083157756, lon=10.164413452148438 },
+        { lat=53.73104466704585,  lon=10.202865600585938 },
+        { lat=53.676781546441546, lon=10.16304016113281  },
+        { lat=53.632832079199474, lon=10.235824584960938 },
+        { lat=53.608803292930894, lon=10.2008056640625   },
+        { lat=53.578646152866504, lon=10.208358764648438 },
+        { lat=53.57212285981298,  lon=10.163726806640625 },
+        { lat=53.52071674896369,  lon=10.18707275390625  },
+        { lat=53.52643162253097,  lon=10.224151611328125 },
+        { lat=53.44062753992289,  lon=10.347747802734375 },
+        { lat=53.38824275010831,  lon=10.248870849609375 },
+        { lat=53.38824275010831,  lon=10.15960693359375  },
+        { lat=53.44635321212876,  lon=10.064849853515625 },
+        { lat=53.40595029739904,  lon=9.985198974609375  },
+        { lat=53.42385506057106,  lon=9.951210021972656  },
+        { lat=53.41843327091211,  lon=9.944171905517578  },
+        { lat=53.41812635648326,  lon=9.927349090576172  },
+        { lat=53.412294561442884, lon=9.917736053466797  },
+        { lat=53.41464783813818,  lon=9.901256561279297  },
+        { lat=53.443490472483326, lon=9.912586212158201  },
+        { lat=53.45177144115704,  lon=9.897651672363281  },
+        { lat=53.43633277935392,  lon=9.866924285888672  },
+        { lat=53.427639673754776, lon=9.866409301757812  },
+        { lat=53.427639673754776, lon=9.858856201171875  },
+        { lat=53.46710230573499,  lon=9.795513153076172  },
+        { lat=53.49039461941655,  lon=9.795341491699219  },
+        { lat=53.49029248806277,  lon=9.77903366088867   },
+        { lat=53.49856433088649,  lon=9.780235290527344  },
+        { lat=53.5078554643033,   lon=9.758434295654297  },
+        { lat=53.545407634092975, lon=9.759807586669922  },
+        { lat=53.568147234570084, lon=9.633293151855469  },
+        { lat=53.58802162343514,  lon=9.655780792236328  },
+        { lat=53.568351121879815, lon=9.727706909179688  },
+        { lat=53.60921067445695,  lon=9.737663269042969  },
+    }
+
+    points = {
+        {
+            name='Alice',
+            lat=52.52437,
+            lon=13.41053,
+        }, {
+            name='Bob',
+            lat=53.551085,
+            lon=9.993682,
+        }, {
+            name='Charlie',
+            lat=40.726662,
+            lon=-74.036677,
+        }
+    }
+
+    for i=1, #points do
+        if geo_polygon_contains(hamburg, points[i]) then
+            info('[INSIDE ] ' .. points[i]['name'])
+        else
+            info('[OUTSIDE] ' .. points[i]['name'])
+        end
+    end
+end

modules/harness/google-tls.lua

@@ -0,0 +1,29 @@
+-- Description: Test various tls functions
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('sending https request to google.com')
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://google.com/', {})
+    r = http_send(req)
+    if last_err() then return end
+    debug(r)
+
+    info('creating tls socket to google.com')
+    sock = sock_connect('google.com', 443, {
+        tls=true,
+    })
+    if last_err() then return end
+    debug(sock)
+
+    info('creating socket to google.com, wrapping afterwards')
+    sock = sock_connect('google.com', 443, {})
+    if last_err() then return end
+    tls = sock_upgrade_tls(sock, {
+        sni_value='google.com',
+    })
+    if last_err() then return end
+    debug(sock)
+    debug(tls)
+end

modules/harness/img-hash.lua

@@ -0,0 +1,11 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    debug(arg)
+    info(img_ahash(arg['value']))
+    info(img_dhash(arg['value']))
+    info(img_phash(arg['value']))
+end

modules/harness/import-subdomains.lua

@@ -0,0 +1,29 @@
+-- Description: Import subdomains from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lua
+
+    while true do
+        local line = stdin_readline()
+        if line == nil then
+            break
+        end
+        -- strip newline
+        local m = regex_find('.+', line)
+        if m then
+            local subdomain = m[1]
+
+            local domain = psl_domain_from_dns_name(subdomain)
+            local domain_id = db_add('domain', {
+                value=domain,
+            })
+
+            db_add('subdomain', {
+                domain_id=domain_id,
+                value=subdomain,
+            })
+        end
+    end
+end

modules/harness/notify-ohai.lua

@@ -0,0 +1,8 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/notify-ratelimit.lua

@@ -0,0 +1,9 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    ratelimit_throttle('foo', 3, 10000)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/notify-slow.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    -- TODO your code here
+    sleep(1)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/onion-http.lua

@@ -0,0 +1,17 @@
+-- Description: Send a request to a hidden service
+-- Version: 0.1.0
+-- Stealth: passive
+-- License: GPL-3.0
+
+function run()
+    local session = http_mksession()
+    local req = http_request(session, 'GET', 'http://expyuzz4wqqyqhjn.onion/', {
+        proxy='127.0.0.1:9050',
+    })
+    local r = http_fetch(req)
+    if last_err() then return end
+
+    local title = html_select(r['text'], 'title')
+    if last_err() then return end
+    info(title['html'])
+end

modules/harness/pgp-fetch.lua

@@ -0,0 +1,22 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+    url = 'https://openpgpkey.archlinux.org/.well-known/openpgpkey/archlinux.org/hu/in9mwr4s84x7gm51851h343n3at1x61g?l=anthraxx'
+
+    req = http_request(session, 'GET', url, {})
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['text'])
+    info(k)
+
+    req = http_request(session, 'GET', url, {
+        binary=true,
+    })
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['binary'])
+    info(k)
+end

modules/harness/pgp-stdin.lua

@@ -0,0 +1,9 @@
+-- Description: Decode armored pgp key from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local pubkey = stdin_read_to_end()
+    pubkey = pgp_pubkey_armored(pubkey)
+    info(pubkey)
+end

modules/harness/port.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    ip_addr = '192.168.1.2'
+    ip_addr_id = db_add('ipaddr', {
+        value=ip_addr,
+    })
+    db_add('port', {
+        ip_addr_id=ip_addr_id,
+        ip_addr=ip_addr,
+        protocol='tcp',
+        port=4444,
+        status='open',
+    })
+end

modules/harness/ratelimit.lua

@@ -0,0 +1,11 @@
+-- Description: Run script with a global ratelimit
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- this shouldn't complete in less than 5 seconds
+    for i=1, 20 do
+        ratelimit_throttle('foo', 4, 1000)
+        info(sn0int_time())
+    end
+end